Merge pull request #4469 from takluyver/py3-getcwdu

Python 3 & getcwdu

We were using the os.getcwdu() function in several places. That doesn't exist on Python 3, but the path.py module was masking that by setting os.getcwdu = os.getcwd. As described in  #4462, however, that didn't always work.

This adds a reference in py3compat which points to getcwd on Python 3 and getcwdu in Python 2.

IPython/core/application.py

@@ -1,381 +1,382 b''
 # encoding: utf-8
 """
 An application for IPython.
 
 All top-level applications should use the classes in this module for
 handling configuration and creating configurables.
 
 The job of an :class:`Application` is to create the master configuration
 object and then create the configurable objects, passing the config to them.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 * Min RK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import atexit
 import errno
 import glob
 import logging
 import os
 import shutil
 import sys
 
 from IPython.config.application import Application, catch_config_error
 from IPython.config.loader import ConfigFileNotFound
 from IPython.core import release, crashhandler
 from IPython.core.profiledir import ProfileDir, ProfileDirError
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
+from IPython.utils import py3compat
 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict, Set, Instance
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 
 #-----------------------------------------------------------------------------
 # Base Application Class
 #-----------------------------------------------------------------------------
 
 # aliases and flags
 
 base_aliases = {
     'profile-dir' : 'ProfileDir.location',
     'profile' : 'BaseIPythonApplication.profile',
     'ipython-dir' : 'BaseIPythonApplication.ipython_dir',
     'log-level' : 'Application.log_level',
     'config' : 'BaseIPythonApplication.extra_config_file',
 }
 
 base_flags = dict(
     debug = ({'Application' : {'log_level' : logging.DEBUG}},
             "set log level to logging.DEBUG (maximize logging output)"),
     quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
             "set log level to logging.CRITICAL (minimize logging output)"),
     init = ({'BaseIPythonApplication' : {
                     'copy_config_files' : True,
                     'auto_create' : True}
             }, """Initialize profile with default config files.  This is equivalent
             to running `ipython profile create <profile>` prior to startup.
             """)
 )
 
 
 class BaseIPythonApplication(Application):
 
     name = Unicode(u'ipython')
     description = Unicode(u'IPython: an enhanced interactive Python shell.')
     version = Unicode(release.version)
 
     aliases = Dict(base_aliases)
     flags = Dict(base_flags)
     classes = List([ProfileDir])
 
     # Track whether the config_file has changed,
     # because some logic happens only if we aren't using the default.
     config_file_specified = Set()
 
     config_file_name = Unicode()
     def _config_file_name_default(self):
         return self.name.replace('-','_') + u'_config.py'
     def _config_file_name_changed(self, name, old, new):
         if new != old:
             self.config_file_specified.add(new)
 
     # The directory that contains IPython's builtin profiles.
     builtin_profile_dir = Unicode(
         os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
     )
 
     config_file_paths = List(Unicode)
     def _config_file_paths_default(self):
-        return [os.getcwdu()]
+        return [py3compat.getcwd()]
 
     extra_config_file = Unicode(config=True,
     help="""Path to an extra config file to load.
     
     If specified, load this config file in addition to any other IPython config.
     """)
     def _extra_config_file_changed(self, name, old, new):
         try:
             self.config_files.remove(old)
         except ValueError:
             pass
         self.config_file_specified.add(new)
         self.config_files.append(new)
 
     profile = Unicode(u'default', config=True,
         help="""The IPython profile to use."""
     )
 
     def _profile_changed(self, name, old, new):
         self.builtin_profile_dir = os.path.join(
                 get_ipython_package_dir(), u'config', u'profile', new
         )
 
     ipython_dir = Unicode(config=True,
         help="""
         The name of the IPython directory. This directory is used for logging
         configuration (through profiles), history storage, etc. The default
         is usually $HOME/.ipython. This options can also be specified through
         the environment variable IPYTHONDIR.
         """
     )
     def _ipython_dir_default(self):
         d = get_ipython_dir()
         self._ipython_dir_changed('ipython_dir', d, d)
         return d
     
     _in_init_profile_dir = False
     profile_dir = Instance(ProfileDir)
     def _profile_dir_default(self):
         # avoid recursion
         if self._in_init_profile_dir:
             return
         # profile_dir requested early, force initialization
         self.init_profile_dir()
         return self.profile_dir
 
     overwrite = Bool(False, config=True,
         help="""Whether to overwrite existing config files when copying""")
     auto_create = Bool(False, config=True,
         help="""Whether to create profile dir if it doesn't exist""")
 
     config_files = List(Unicode)
     def _config_files_default(self):
         return [self.config_file_name]
 
     copy_config_files = Bool(False, config=True,
         help="""Whether to install the default config files into the profile dir.
         If a new profile is being created, and IPython contains config files for that
         profile, then they will be staged into the new directory.  Otherwise,
         default config files will be automatically generated.
         """)
     
     verbose_crash = Bool(False, config=True,
         help="""Create a massive crash report when IPython encounters what may be an
         internal error.  The default is to append a short message to the
         usual traceback""")
 
     # The class to use as the crash handler.
     crash_handler_class = Type(crashhandler.CrashHandler)
 
     @catch_config_error
     def __init__(self, **kwargs):
         super(BaseIPythonApplication, self).__init__(**kwargs)
         # ensure current working directory exists
         try:
-            directory = os.getcwdu()
+            directory = py3compat.getcwd()
         except:
             # raise exception
             self.log.error("Current working directory doesn't exist.")
             raise
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
     def init_crash_handler(self):
         """Create a crash handler, typically setting sys.excepthook to it."""
         self.crash_handler = self.crash_handler_class(self)
         sys.excepthook = self.excepthook
         def unset_crashhandler():
             sys.excepthook = sys.__excepthook__
         atexit.register(unset_crashhandler)
     
     def excepthook(self, etype, evalue, tb):
         """this is sys.excepthook after init_crashhandler
         
         set self.verbose_crash=True to use our full crashhandler, instead of
         a regular traceback with a short message (crash_handler_lite)
         """
         
         if self.verbose_crash:
             return self.crash_handler(etype, evalue, tb)
         else:
             return crashhandler.crash_handler_lite(etype, evalue, tb)
     
     def _ipython_dir_changed(self, name, old, new):
         if old in sys.path:
             sys.path.remove(old)
         sys.path.append(os.path.abspath(new))
         if not os.path.isdir(new):
             os.makedirs(new, mode=0o777)
         readme = os.path.join(new, 'README')
         readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README')
         if not os.path.exists(readme) and os.path.exists(readme_src):
             shutil.copy(readme_src, readme)
         for d in ('extensions', 'nbextensions'):
             path = os.path.join(new, d)
             if not os.path.exists(path):
                 try:
                     os.mkdir(path)
                 except OSError as e:
                     if e.errno != errno.EEXIST:
                         self.log.error("couldn't create path %s: %s", path, e)
         self.log.debug("IPYTHONDIR set to: %s" % new)
 
     def load_config_file(self, suppress_errors=True):
         """Load the config file.
 
         By default, errors in loading config are handled, and a warning
         printed on screen. For testing, the suppress_errors option is set
         to False, so errors will make tests fail.
         """
         self.log.debug("Searching path %s for config files", self.config_file_paths)
         base_config = 'ipython_config.py'
         self.log.debug("Attempting to load config file: %s" %
                        base_config)
         try:
             Application.load_config_file(
                 self,
                 base_config,
                 path=self.config_file_paths
             )
         except ConfigFileNotFound:
             # ignore errors loading parent
             self.log.debug("Config file %s not found", base_config)
             pass
         
         for config_file_name in self.config_files:
             if not config_file_name or config_file_name == base_config:
                 continue
             self.log.debug("Attempting to load config file: %s" %
                            self.config_file_name)
             try:
                 Application.load_config_file(
                     self,
                     config_file_name,
                     path=self.config_file_paths
                 )
             except ConfigFileNotFound:
                 # Only warn if the default config file was NOT being used.
                 if config_file_name in self.config_file_specified:
                     msg = self.log.warn
                 else:
                     msg = self.log.debug
                 msg("Config file not found, skipping: %s", config_file_name)
             except:
                 # For testing purposes.
                 if not suppress_errors:
                     raise
                 self.log.warn("Error loading config file: %s" %
                               self.config_file_name, exc_info=True)
 
     def init_profile_dir(self):
         """initialize the profile dir"""
         self._in_init_profile_dir = True
         if self.profile_dir is not None:
             # already ran
             return
         if 'ProfileDir.location' not in self.config:
             # location not specified, find by profile name
             try:
                 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
             except ProfileDirError:
                 # not found, maybe create it (always create default profile)
                 if self.auto_create or self.profile == 'default':
                     try:
                         p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
                     except ProfileDirError:
                         self.log.fatal("Could not create profile: %r"%self.profile)
                         self.exit(1)
                     else:
                         self.log.info("Created profile dir: %r"%p.location)
                 else:
                     self.log.fatal("Profile %r not found."%self.profile)
                     self.exit(1)
             else:
                 self.log.info("Using existing profile dir: %r"%p.location)
         else:
             location = self.config.ProfileDir.location
             # location is fully specified
             try:
                 p = ProfileDir.find_profile_dir(location, self.config)
             except ProfileDirError:
                 # not found, maybe create it
                 if self.auto_create:
                     try:
                         p = ProfileDir.create_profile_dir(location, self.config)
                     except ProfileDirError:
                         self.log.fatal("Could not create profile directory: %r"%location)
                         self.exit(1)
                     else:
                         self.log.info("Creating new profile dir: %r"%location)
                 else:
                     self.log.fatal("Profile directory %r not found."%location)
                     self.exit(1)
             else:
                 self.log.info("Using existing profile dir: %r"%location)
 
         self.profile_dir = p
         self.config_file_paths.append(p.location)
         self._in_init_profile_dir = False
 
     def init_config_files(self):
         """[optionally] copy default config files into profile dir."""
         # copy config files
         path = self.builtin_profile_dir
         if self.copy_config_files:
             src = self.profile
 
             cfg = self.config_file_name
             if path and os.path.exists(os.path.join(path, cfg)):
                 self.log.warn("Staging %r from %s into %r [overwrite=%s]"%(
                         cfg, src, self.profile_dir.location, self.overwrite)
                 )
                 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
             else:
                 self.stage_default_config_file()
         else:
             # Still stage *bundled* config files, but not generated ones
             # This is necessary for `ipython profile=sympy` to load the profile
             # on the first go
             files = glob.glob(os.path.join(path, '*.py'))
             for fullpath in files:
                 cfg = os.path.basename(fullpath)
                 if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
                     # file was copied
                     self.log.warn("Staging bundled %s from %s into %r"%(
                             cfg, self.profile, self.profile_dir.location)
                     )
 
 
     def stage_default_config_file(self):
         """auto generate default config file, and stage it into the profile."""
         s = self.generate_config_file()
         fname = os.path.join(self.profile_dir.location, self.config_file_name)
         if self.overwrite or not os.path.exists(fname):
             self.log.warn("Generating default config file: %r"%(fname))
             with open(fname, 'w') as f:
                 f.write(s)
 
     @catch_config_error
     def initialize(self, argv=None):
         # don't hook up crash handler before parsing command-line
         self.parse_command_line(argv)
         self.init_crash_handler()
         if self.subapp is not None:
             # stop here if subapp is taking over
             return
         cl_config = self.config
         self.init_profile_dir()
         self.init_config_files()
         self.load_config_file()
         # enforce cl-opts override configfile opts:
         self.update_config(cl_config)
 

IPython/core/crashhandler.py

@@ -1,216 +1,216 b''
 # encoding: utf-8
 """sys.excepthook for IPython itself, leaves a detailed report on disk.
 
 Authors:
 
 * Fernando Perez
 * Brian E. Granger
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import os
 import sys
 import traceback
 from pprint import pformat
 
 from IPython.core import ultratb
 from IPython.core.release import author_email
 from IPython.utils.sysinfo import sys_info
-from IPython.utils.py3compat import input
+from IPython.utils.py3compat import input, getcwd
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 # Template for the user message.
 _default_message_template = """\
 Oops, {app_name} crashed. We do our best to make it stable, but...
 
 A crash report was automatically generated with the following information:
   - A verbatim copy of the crash traceback.
   - A copy of your input history during this session.
   - Data on your current {app_name} configuration.
 
 It was left in the file named:
 \t'{crash_report_fname}'
 If you can email this file to the developers, the information in it will help
 them in understanding and correcting the problem.
 
 You can mail it to: {contact_name} at {contact_email}
 with the subject '{app_name} Crash Report'.
 
 If you want to do it now, the following command will work (under Unix):
 mail -s '{app_name} Crash Report' {contact_email} < {crash_report_fname}
 
 To ensure accurate tracking of this issue, please file a report about it at:
 {bug_tracker}
 """
 
 _lite_message_template = """
 If you suspect this is an IPython bug, please report it at:
     https://github.com/ipython/ipython/issues
 or send an email to the mailing list at {email}
 
 You can print a more detailed traceback right now with "%tb", or use "%debug"
 to interactively debug it.
 
 Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
     {config}Application.verbose_crash=True
 """
 
 
 class CrashHandler(object):
     """Customizable crash handlers for IPython applications.
 
     Instances of this class provide a :meth:`__call__` method which can be
     used as a ``sys.excepthook``.  The :meth:`__call__` signature is::
 
         def __call__(self, etype, evalue, etb)
     """
 
     message_template = _default_message_template
     section_sep = '\n\n'+'*'*75+'\n\n'
 
     def __init__(self, app, contact_name=None, contact_email=None,
                  bug_tracker=None, show_crash_traceback=True, call_pdb=False):
         """Create a new crash handler
 
         Parameters
         ----------
         app :  Application
             A running :class:`Application` instance, which will be queried at
             crash time for internal information.
 
         contact_name : str
             A string with the name of the person to contact.
 
         contact_email : str
             A string with the email address of the contact.
 
         bug_tracker : str
             A string with the URL for your project's bug tracker.
 
         show_crash_traceback : bool
             If false, don't print the crash traceback on stderr, only generate
             the on-disk report
 
         Non-argument instance attributes:
 
         These instances contain some non-argument attributes which allow for
         further customization of the crash handler's behavior. Please see the
         source for further details.
         """
         self.crash_report_fname = "Crash_report_%s.txt" % app.name
         self.app = app
         self.call_pdb = call_pdb
         #self.call_pdb = True # dbg
         self.show_crash_traceback = show_crash_traceback
         self.info = dict(app_name = app.name,
                     contact_name = contact_name,
                     contact_email = contact_email,
                     bug_tracker = bug_tracker,
                     crash_report_fname = self.crash_report_fname)
 
 
     def __call__(self, etype, evalue, etb):
         """Handle an exception, call for compatible with sys.excepthook"""
         
         # do not allow the crash handler to be called twice without reinstalling it
         # this prevents unlikely errors in the crash handling from entering an
         # infinite loop.
         sys.excepthook = sys.__excepthook__
         
         # Report tracebacks shouldn't use color in general (safer for users)
         color_scheme = 'NoColor'
 
         # Use this ONLY for developer debugging (keep commented out for release)
         #color_scheme = 'Linux'   # dbg
         try:
             rptdir = self.app.ipython_dir
         except:
-            rptdir = os.getcwdu()
+            rptdir = getcwd()
         if rptdir is None or not os.path.isdir(rptdir):
-            rptdir = os.getcwdu()
+            rptdir = getcwd()
         report_name = os.path.join(rptdir,self.crash_report_fname)
         # write the report filename into the instance dict so it can get
         # properly expanded out in the user message template
         self.crash_report_fname = report_name
         self.info['crash_report_fname'] = report_name
         TBhandler = ultratb.VerboseTB(
             color_scheme=color_scheme,
             long_header=1,
             call_pdb=self.call_pdb,
         )
         if self.call_pdb:
             TBhandler(etype,evalue,etb)
             return
         else:
             traceback = TBhandler.text(etype,evalue,etb,context=31)
 
         # print traceback to screen
         if self.show_crash_traceback:
             print(traceback, file=sys.stderr)
 
         # and generate a complete report on disk
         try:
             report = open(report_name,'w')
         except:
             print('Could not create crash report on disk.', file=sys.stderr)
             return
 
         # Inform user on stderr of what happened
         print('\n'+'*'*70+'\n', file=sys.stderr)
         print(self.message_template.format(**self.info), file=sys.stderr)
 
         # Construct report on disk
         report.write(self.make_report(traceback))
         report.close()
         input("Hit <Enter> to quit (your terminal may close):")
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
 
         sec_sep = self.section_sep
 
         report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n']
         rpt_add = report.append
         rpt_add(sys_info())
 
         try:
             config = pformat(self.app.config)
             rpt_add(sec_sep)
             rpt_add('Application name: %s\n\n' % self.app_name)
             rpt_add('Current user configuration structure:\n\n')
             rpt_add(config)
         except:
             pass
         rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
 
         return ''.join(report)
 
 
 def crash_handler_lite(etype, evalue, tb):
     """a light excepthook, adding a small message to the usual traceback"""
     traceback.print_exception(etype, evalue, tb)
     
     from IPython.core.interactiveshell import InteractiveShell
     if InteractiveShell.initialized():
         # we are in a Shell environment, give %magic example
         config = "%config "
     else:
         # we are not in a shell, show generic config
         config = "c."
     print(_lite_message_template.format(email=author_email, config=config), file=sys.stderr)
 

IPython/core/history.py

@@ -1,807 +1,808 b''
 """ History related magics and functionality """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011 The IPython Development Team.
 #
 #  Distributed under the terms of the BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib imports
 import atexit
 import datetime
 import os
 import re
 try:
     import sqlite3
 except ImportError:
     try:
         from pysqlite2 import dbapi2 as sqlite3
     except ImportError:
         sqlite3 = None
 import threading
 
 # Our own packages
 from IPython.config.configurable import Configurable
 from IPython.external.decorator import decorator
 from IPython.utils.path import locate_profile
+from IPython.utils import py3compat
 from IPython.utils.traitlets import (
     Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
 )
 from IPython.utils.warn import warn
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 class DummyDB(object):
     """Dummy DB that will act as a black hole for history.
     
     Only used in the absence of sqlite"""
     def execute(*args, **kwargs):
         return []
     
     def commit(self, *args, **kwargs):
         pass
     
     def __enter__(self, *args, **kwargs):
         pass
     
     def __exit__(self, *args, **kwargs):
         pass
 
 
 @decorator
 def needs_sqlite(f, self, *a, **kw):
     """return an empty list in the absence of sqlite"""
     if sqlite3 is None or not self.enabled:
         return []
     else:
         return f(self, *a, **kw)
 
 
 if sqlite3 is not None:
     DatabaseError = sqlite3.DatabaseError
 else:
     class DatabaseError(Exception):
         "Dummy exception when sqlite could not be imported. Should never occur."
 
 @decorator
 def catch_corrupt_db(f, self, *a, **kw):
     """A decorator which wraps HistoryAccessor method calls to catch errors from
     a corrupt SQLite database, move the old database out of the way, and create
     a new one.
     """
     try:
         return f(self, *a, **kw)
     except DatabaseError:
         if os.path.isfile(self.hist_file):
             # Try to move the file out of the way
             base,ext = os.path.splitext(self.hist_file)
             newpath = base + '-corrupt' + ext
             os.rename(self.hist_file, newpath)
             self.init_db()
             print("ERROR! History file wasn't a valid SQLite database.",
             "It was moved to %s" % newpath, "and a new file created.")
             return []
         
         else:
             # The hist_file is probably :memory: or something else.
             raise
         
 
 
 class HistoryAccessor(Configurable):
     """Access the history database without adding to it.
     
     This is intended for use by standalone history tools. IPython shells use
     HistoryManager, below, which is a subclass of this."""
 
     # String holding the path to the history file
     hist_file = Unicode(config=True,
         help="""Path to file to use for SQLite history database.
         
         By default, IPython will put the history database in the IPython
         profile directory.  If you would rather share one history among
         profiles, you can set this value in each, so that they are consistent.
         
         Due to an issue with fcntl, SQLite is known to misbehave on some NFS
         mounts.  If you see IPython hanging, try setting this to something on a
         local disk, e.g::
         
             ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
         
         """)
     
     enabled = Bool(True, config=True,
         help="""enable the SQLite history
         
         set enabled=False to disable the SQLite history,
         in which case there will be no stored history, no SQLite connection,
         and no background saving thread.  This may be necessary in some
         threaded environments where IPython is embedded.
         """
     )
     
     connection_options = Dict(config=True,
         help="""Options for configuring the SQLite connection
         
         These options are passed as keyword args to sqlite3.connect
         when establishing database conenctions.
         """
     )
 
     # The SQLite database
     db = Any()
     def _db_changed(self, name, old, new):
         """validate the db, since it can be an Instance of two different types"""
         connection_types = (DummyDB,)
         if sqlite3 is not None:
             connection_types = (DummyDB, sqlite3.Connection)
         if not isinstance(new, connection_types):
             msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
                     (self.__class__.__name__, new)
             raise TraitError(msg)
     
     def __init__(self, profile='default', hist_file=u'', **traits):
         """Create a new history accessor.
         
         Parameters
         ----------
         profile : str
           The name of the profile from which to open history.
         hist_file : str
           Path to an SQLite history database stored by IPython. If specified,
           hist_file overrides profile.
         config :
           Config object. hist_file can also be set through this.
         """
         # We need a pointer back to the shell for various tasks.
         super(HistoryAccessor, self).__init__(**traits)
         # defer setting hist_file from kwarg until after init,
         # otherwise the default kwarg value would clobber any value
         # set by config
         if hist_file:
             self.hist_file = hist_file
         
         if self.hist_file == u'':
             # No one has set the hist_file, yet.
             self.hist_file = self._get_hist_file_name(profile)
 
         if sqlite3 is None and self.enabled:
             warn("IPython History requires SQLite, your history will not be saved")
             self.enabled = False
         
         self.init_db()
     
     def _get_hist_file_name(self, profile='default'):
         """Find the history file for the given profile name.
         
         This is overridden by the HistoryManager subclass, to use the shell's
         active profile.
         
         Parameters
         ----------
         profile : str
           The name of a profile which has a history file.
         """
         return os.path.join(locate_profile(profile), 'history.sqlite')
     
     @catch_corrupt_db
     def init_db(self):
         """Connect to the database, and create tables if necessary."""
         if not self.enabled:
             self.db = DummyDB()
             return
         
         # use detect_types so that timestamps return datetime objects
         kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
         kwargs.update(self.connection_options)
         self.db = sqlite3.connect(self.hist_file, **kwargs)
         self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
                         primary key autoincrement, start timestamp,
                         end timestamp, num_cmds integer, remark text)""")
         self.db.execute("""CREATE TABLE IF NOT EXISTS history
                 (session integer, line integer, source text, source_raw text,
                 PRIMARY KEY (session, line))""")
         # Output history is optional, but ensure the table's there so it can be
         # enabled later.
         self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
                         (session integer, line integer, output text,
                         PRIMARY KEY (session, line))""")
         self.db.commit()
 
     def writeout_cache(self):
         """Overridden by HistoryManager to dump the cache before certain
         database lookups."""
         pass
 
     ## -------------------------------
     ## Methods for retrieving history:
     ## -------------------------------
     def _run_sql(self, sql, params, raw=True, output=False):
         """Prepares and runs an SQL query for the history database.
 
         Parameters
         ----------
         sql : str
           Any filtering expressions to go after SELECT ... FROM ...
         params : tuple
           Parameters passed to the SQL query (to replace "?")
         raw, output : bool
           See :meth:`get_range`
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         toget = 'source_raw' if raw else 'source'
         sqlfrom = "history"
         if output:
             sqlfrom = "history LEFT JOIN output_history USING (session, line)"
             toget = "history.%s, output_history.output" % toget
         cur = self.db.execute("SELECT session, line, %s FROM %s " %\
                                 (toget, sqlfrom) + sql, params)
         if output:    # Regroup into 3-tuples, and parse JSON
             return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
         return cur
 
     @needs_sqlite
     @catch_corrupt_db
     def get_session_info(self, session=0):
         """get info about a session
 
         Parameters
         ----------
 
         session : int
             Session number to retrieve. The current session is 0, and negative
             numbers count back from current session, so -1 is previous session.
 
         Returns
         -------
 
         (session_id [int], start [datetime], end [datetime], num_cmds [int],
         remark [unicode])
 
         Sessions that are running or did not exit cleanly will have `end=None`
         and `num_cmds=None`.
 
         """
 
         if session <= 0:
             session += self.session_number
 
         query = "SELECT * from sessions where session == ?"
         return self.db.execute(query, (session,)).fetchone()
 
     @catch_corrupt_db
     def get_tail(self, n=10, raw=True, output=False, include_latest=False):
         """Get the last n lines from the history database.
 
         Parameters
         ----------
         n : int
           The number of lines to get
         raw, output : bool
           See :meth:`get_range`
         include_latest : bool
           If False (default), n+1 lines are fetched, and the latest one
           is discarded. This is intended to be used where the function
           is called by a user command, which it should not return.
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         self.writeout_cache()
         if not include_latest:
             n += 1
         cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
                                 (n,), raw=raw, output=output)
         if not include_latest:
             return reversed(list(cur)[1:])
         return reversed(list(cur))
 
     @catch_corrupt_db
     def search(self, pattern="*", raw=True, search_raw=True,
                output=False, n=None, unique=False):
         """Search the database using unix glob-style matching (wildcards
         * and ?).
 
         Parameters
         ----------
         pattern : str
           The wildcarded pattern to match when searching
         search_raw : bool
           If True, search the raw input, otherwise, the parsed input
         raw, output : bool
           See :meth:`get_range`
         n : None or int
           If an integer is given, it defines the limit of
           returned entries.
         unique : bool
           When it is true, return only unique entries.
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         tosearch = "source_raw" if search_raw else "source"
         if output:
             tosearch = "history." + tosearch
         self.writeout_cache()
         sqlform = "WHERE %s GLOB ?" % tosearch
         params = (pattern,)
         if unique:
             sqlform += ' GROUP BY {0}'.format(tosearch)
         if n is not None:
             sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
             params += (n,)
         elif unique:
             sqlform += " ORDER BY session, line"
         cur = self._run_sql(sqlform, params, raw=raw, output=output)
         if n is not None:
             return reversed(list(cur))
         return cur
     
     @catch_corrupt_db
     def get_range(self, session, start=1, stop=None, raw=True,output=False):
         """Retrieve input by session.
 
         Parameters
         ----------
         session : int
             Session number to retrieve.
         start : int
             First line to retrieve.
         stop : int
             End of line range (excluded from output itself). If None, retrieve
             to the end of the session.
         raw : bool
             If True, return untranslated input
         output : bool
             If True, attempt to include output. This will be 'real' Python
             objects for the current session, or text reprs from previous
             sessions if db_log_output was enabled at the time. Where no output
             is found, None is used.
 
         Returns
         -------
         An iterator over the desired lines. Each line is a 3-tuple, either
         (session, line, input) if output is False, or
         (session, line, (input, output)) if output is True.
         """
         if stop:
             lineclause = "line >= ? AND line < ?"
             params = (session, start, stop)
         else:
             lineclause = "line>=?"
             params = (session, start)
 
         return self._run_sql("WHERE session==? AND %s" % lineclause,
                                     params, raw=raw, output=output)
 
     def get_range_by_str(self, rangestr, raw=True, output=False):
         """Get lines of history from a string of ranges, as used by magic
         commands %hist, %save, %macro, etc.
 
         Parameters
         ----------
         rangestr : str
           A string specifying ranges, e.g. "5 ~2/1-4". See
           :func:`magic_history` for full details.
         raw, output : bool
           As :meth:`get_range`
 
         Returns
         -------
         Tuples as :meth:`get_range`
         """
         for sess, s, e in extract_hist_ranges(rangestr):
             for line in self.get_range(sess, s, e, raw=raw, output=output):
                 yield line
 
 
 class HistoryManager(HistoryAccessor):
     """A class to organize all history-related functionality in one place.
     """
     # Public interface
 
     # An instance of the IPython shell we are attached to
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     # Lists to hold processed and raw history. These start with a blank entry
     # so that we can index them starting from 1
     input_hist_parsed = List([""])
     input_hist_raw = List([""])
     # A list of directories visited during session
     dir_hist = List()
     def _dir_hist_default(self):
         try:
-            return [os.getcwdu()]
+            return [py3compat.getcwd()]
         except OSError:
             return []
 
     # A dict of output history, keyed with ints from the shell's
     # execution count.
     output_hist = Dict()
     # The text/plain repr of outputs.
     output_hist_reprs = Dict()
 
     # The number of the current session in the history database
     session_number = Integer()
     # Should we log output to the database? (default no)
     db_log_output = Bool(False, config=True)
     # Write to database every x commands (higher values save disk access & power)
     #  Values of 1 or less effectively disable caching. 
     db_cache_size = Integer(0, config=True)
     # The input and output caches
     db_input_cache = List()
     db_output_cache = List()
     
     # History saving in separate thread
     save_thread = Instance('IPython.core.history.HistorySavingThread')
     try:               # Event is a function returning an instance of _Event...
         save_flag = Instance(threading._Event)
     except AttributeError:         # ...until Python 3.3, when it's a class.
         save_flag = Instance(threading.Event)
     
     # Private interface
     # Variables used to store the three last inputs from the user.  On each new
     # history update, we populate the user's namespace with these, shifted as
     # necessary.
     _i00 = Unicode(u'')
     _i = Unicode(u'')
     _ii = Unicode(u'')
     _iii = Unicode(u'')
 
     # A regex matching all forms of the exit command, so that we don't store
     # them in the history (it's annoying to rewind the first entry and land on
     # an exit call).
     _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
 
     def __init__(self, shell=None, config=None, **traits):
         """Create a new history manager associated with a shell instance.
         """
         # We need a pointer back to the shell for various tasks.
         super(HistoryManager, self).__init__(shell=shell, config=config,
             **traits)
         self.save_flag = threading.Event()
         self.db_input_cache_lock = threading.Lock()
         self.db_output_cache_lock = threading.Lock()
         if self.enabled and self.hist_file != ':memory:':
             self.save_thread = HistorySavingThread(self)
             self.save_thread.start()
 
         self.new_session()
 
     def _get_hist_file_name(self, profile=None):
         """Get default history file name based on the Shell's profile.
         
         The profile parameter is ignored, but must exist for compatibility with
         the parent class."""
         profile_dir = self.shell.profile_dir.location
         return os.path.join(profile_dir, 'history.sqlite')
     
     @needs_sqlite
     def new_session(self, conn=None):
         """Get a new session number."""
         if conn is None:
             conn = self.db
         
         with conn:
             cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
                             NULL, "") """, (datetime.datetime.now(),))
             self.session_number = cur.lastrowid
             
     def end_session(self):
         """Close the database session, filling in the end time and line count."""
         self.writeout_cache()
         with self.db:
             self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
                             session==?""", (datetime.datetime.now(),
                             len(self.input_hist_parsed)-1, self.session_number))
         self.session_number = 0
                             
     def name_session(self, name):
         """Give the current session a name in the history database."""
         with self.db:
             self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
                             (name, self.session_number))
                             
     def reset(self, new_session=True):
         """Clear the session history, releasing all object references, and
         optionally open a new session."""
         self.output_hist.clear()
         # The directory history can't be completely empty
-        self.dir_hist[:] = [os.getcwdu()]
+        self.dir_hist[:] = [py3compat.getcwd()]
         
         if new_session:
             if self.session_number:
                 self.end_session()
             self.input_hist_parsed[:] = [""]
             self.input_hist_raw[:] = [""]
             self.new_session()
     
     # ------------------------------
     # Methods for retrieving history
     # ------------------------------
     def _get_range_session(self, start=1, stop=None, raw=True, output=False):
         """Get input and output history from the current session. Called by
         get_range, and takes similar parameters."""
         input_hist = self.input_hist_raw if raw else self.input_hist_parsed
             
         n = len(input_hist)
         if start < 0:
             start += n
         if not stop or (stop > n):
             stop = n
         elif stop < 0:
             stop += n
         
         for i in range(start, stop):
             if output:
                 line = (input_hist[i], self.output_hist_reprs.get(i))
             else:
                 line = input_hist[i]
             yield (0, i, line)
     
     def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
         """Retrieve input by session.
         
         Parameters
         ----------
         session : int
             Session number to retrieve. The current session is 0, and negative
             numbers count back from current session, so -1 is previous session.
         start : int
             First line to retrieve.
         stop : int
             End of line range (excluded from output itself). If None, retrieve
             to the end of the session.
         raw : bool
             If True, return untranslated input
         output : bool
             If True, attempt to include output. This will be 'real' Python
             objects for the current session, or text reprs from previous
             sessions if db_log_output was enabled at the time. Where no output
             is found, None is used.
             
         Returns
         -------
         An iterator over the desired lines. Each line is a 3-tuple, either
         (session, line, input) if output is False, or
         (session, line, (input, output)) if output is True.
         """
         if session <= 0:
             session += self.session_number
         if session==self.session_number:          # Current session
             return self._get_range_session(start, stop, raw, output)
         return super(HistoryManager, self).get_range(session, start, stop, raw,
                                                      output)
 
     ## ----------------------------
     ## Methods for storing history:
     ## ----------------------------
     def store_inputs(self, line_num, source, source_raw=None):
         """Store source and raw input in history and create input cache
         variables _i*.
 
         Parameters
         ----------
         line_num : int
           The prompt number of this input.
 
         source : str
           Python input.
 
         source_raw : str, optional
           If given, this is the raw input without any IPython transformations
           applied to it.  If not given, ``source`` is used.
         """
         if source_raw is None:
             source_raw = source
         source = source.rstrip('\n')
         source_raw = source_raw.rstrip('\n')
 
         # do not store exit/quit commands
         if self._exit_re.match(source_raw.strip()):
             return
 
         self.input_hist_parsed.append(source)
         self.input_hist_raw.append(source_raw)
 
         with self.db_input_cache_lock:
             self.db_input_cache.append((line_num, source, source_raw))
             # Trigger to flush cache and write to DB.
             if len(self.db_input_cache) >= self.db_cache_size:
                 self.save_flag.set()
 
         # update the auto _i variables
         self._iii = self._ii
         self._ii = self._i
         self._i = self._i00
         self._i00 = source_raw
 
         # hackish access to user namespace to create _i1,_i2... dynamically
         new_i = '_i%s' % line_num
         to_main = {'_i': self._i,
                    '_ii': self._ii,
                    '_iii': self._iii,
                    new_i : self._i00 }
         
         if self.shell is not None:
             self.shell.push(to_main, interactive=False)
 
     def store_output(self, line_num):
         """If database output logging is enabled, this saves all the
         outputs from the indicated prompt number to the database. It's
         called by run_cell after code has been executed.
 
         Parameters
         ----------
         line_num : int
           The line number from which to save outputs
         """
         if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
             return
         output = self.output_hist_reprs[line_num]
 
         with self.db_output_cache_lock:
             self.db_output_cache.append((line_num, output))
         if self.db_cache_size <= 1:
             self.save_flag.set()
 
     def _writeout_input_cache(self, conn):
         with conn:
             for line in self.db_input_cache:
                 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
                                 (self.session_number,)+line)
 
     def _writeout_output_cache(self, conn):
         with conn:
             for line in self.db_output_cache:
                 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
                                 (self.session_number,)+line)
 
     @needs_sqlite
     def writeout_cache(self, conn=None):
         """Write any entries in the cache to the database."""
         if conn is None:
             conn = self.db
 
         with self.db_input_cache_lock:
             try:
                 self._writeout_input_cache(conn)
             except sqlite3.IntegrityError:
                 self.new_session(conn)
                 print("ERROR! Session/line number was not unique in",
                       "database. History logging moved to new session",
                                                 self.session_number)
                 try:
                     # Try writing to the new session. If this fails, don't
                     # recurse
                     self._writeout_input_cache(conn)
                 except sqlite3.IntegrityError:
                     pass
             finally:
                 self.db_input_cache = []
 
         with self.db_output_cache_lock:
             try:
                 self._writeout_output_cache(conn)
             except sqlite3.IntegrityError:
                 print("!! Session/line number for output was not unique",
                       "in database. Output will not be stored.")
             finally:
                 self.db_output_cache = []
 
 
 class HistorySavingThread(threading.Thread):
     """This thread takes care of writing history to the database, so that
     the UI isn't held up while that happens.
 
     It waits for the HistoryManager's save_flag to be set, then writes out
     the history cache. The main thread is responsible for setting the flag when
     the cache size reaches a defined threshold."""
     daemon = True
     stop_now = False
     enabled = True
     def __init__(self, history_manager):
         super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
         self.history_manager = history_manager
         self.enabled = history_manager.enabled
         atexit.register(self.stop)
 
     @needs_sqlite
     def run(self):
         # We need a separate db connection per thread:
         try:
             self.db = sqlite3.connect(self.history_manager.hist_file,
                             **self.history_manager.connection_options
             )
             while True:
                 self.history_manager.save_flag.wait()
                 if self.stop_now:
                     return
                 self.history_manager.save_flag.clear()
                 self.history_manager.writeout_cache(self.db)
         except Exception as e:
             print(("The history saving thread hit an unexpected error (%s)."
                    "History will not be written to the database.") % repr(e))
 
     def stop(self):
         """This can be called from the main thread to safely stop this thread.
 
         Note that it does not attempt to write out remaining history before
         exiting. That should be done by calling the HistoryManager's
         end_session method."""
         self.stop_now = True
         self.history_manager.save_flag.set()
         self.join()
 
 
 # To match, e.g. ~5/8-~2/3
 range_re = re.compile(r"""
 ((?P<startsess>~?\d+)/)?
 (?P<start>\d+)?
 ((?P<sep>[\-:])
  ((?P<endsess>~?\d+)/)?
  (?P<end>\d+))?
 $""", re.VERBOSE)
 
 
 def extract_hist_ranges(ranges_str):
     """Turn a string of history ranges into 3-tuples of (session, start, stop).
 
     Examples
     --------
     list(extract_input_ranges("~8/5-~7/4 2"))
     [(-8, 5, None), (-7, 1, 4), (0, 2, 3)]
     """
     for range_str in ranges_str.split():
         rmatch = range_re.match(range_str)
         if not rmatch:
             continue
         start = rmatch.group("start")
         if start:
             start = int(start)
             end = rmatch.group("end")
             # If no end specified, get (a, a + 1)
             end = int(end) if end else start + 1
         else:  # start not specified
             if not rmatch.group('startsess'):  # no startsess
                 continue
             start = 1
             end = None  # provide the entire session hist
 
         if rmatch.group("sep") == "-":       # 1-3 == 1:4 --> [1, 2, 3]
             end += 1
         startsess = rmatch.group("startsess") or "0"
         endsess = rmatch.group("endsess") or startsess
         startsess = int(startsess.replace("~","-"))
         endsess = int(endsess.replace("~","-"))
         assert endsess >= startsess, "start session must be earlier than end session"
 
         if endsess == startsess:
             yield (startsess, start, end)
             continue
         # Multiple sessions in one range:
         yield (startsess, start, None)
         for sess in range(startsess+1, endsess):
             yield (sess, 1, None)
         yield (endsess, 1, end)
 
 
 def _format_lineno(session, line):
     """Helper function to format line numbers properly."""
     if session == 0:
         return str(line)
     return "%s#%s" % (session, line)
 
 

IPython/core/interactiveshell.py

@@ -1,3164 +1,3164 b''
 # -*- coding: utf-8 -*-
 """Main IPython class."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]>
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import absolute_import
 from __future__ import print_function
 
 import __future__
 import abc
 import ast
 import atexit
 import functools
 import os
 import re
 import runpy
 import sys
 import tempfile
 import types
 import subprocess
 from io import open as io_open
 
 from IPython.config.configurable import SingletonConfigurable
 from IPython.core import debugger, oinspect
 from IPython.core import magic
 from IPython.core import page
 from IPython.core import prefilter
 from IPython.core import shadowns
 from IPython.core import ultratb
 from IPython.core.alias import AliasManager, AliasError
 from IPython.core.autocall import ExitAutocall
 from IPython.core.builtin_trap import BuiltinTrap
 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.displayhook import DisplayHook
 from IPython.core.displaypub import DisplayPublisher
 from IPython.core.error import UsageError
 from IPython.core.extensions import ExtensionManager
 from IPython.core.formatters import DisplayFormatter
 from IPython.core.history import HistoryManager
 from IPython.core.inputsplitter import IPythonInputSplitter, ESC_MAGIC, ESC_MAGIC2
 from IPython.core.logger import Logger
 from IPython.core.macro import Macro
 from IPython.core.payload import PayloadManager
 from IPython.core.prefilter import PrefilterManager
 from IPython.core.profiledir import ProfileDir
 from IPython.core.prompts import PromptManager
 from IPython.lib.latextools import LaTeXTool
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import PyColorize
 from IPython.utils import io
 from IPython.utils import py3compat
 from IPython.utils import openpy
 from IPython.utils.decorators import undoc
 from IPython.utils.io import ask_yes_no
 from IPython.utils.ipstruct import Struct
 from IPython.utils.path import get_home_dir, get_ipython_dir, get_py_filename, unquote_filename
 from IPython.utils.pickleshare import PickleShareDB
 from IPython.utils.process import system, getoutput
 from IPython.utils.py3compat import (builtin_mod, unicode_type, string_types,
                                      with_metaclass, iteritems)
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
 from IPython.utils.text import (format_screen, LSString, SList,
                                 DollarFormatter)
 from IPython.utils.traitlets import (Integer, CBool, CaselessStrEnum, Enum,
                                      List, Unicode, Instance, Type)
 from IPython.utils.warn import warn, error
 import IPython.core.hooks
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # compiled regexps for autoindent management
 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 @undoc
 def softspace(file, newvalue):
     """Copied from code.py, to remove the dependency"""
 
     oldvalue = 0
     try:
         oldvalue = file.softspace
     except AttributeError:
         pass
     try:
         file.softspace = newvalue
     except (AttributeError, TypeError):
         # "attribute-less object" or "read-only attributes"
         pass
     return oldvalue
 
 @undoc
 def no_op(*a, **kw): pass
 
 @undoc
 class NoOpContext(object):
     def __enter__(self): pass
     def __exit__(self, type, value, traceback): pass
 no_op_context = NoOpContext()
 
 class SpaceInInput(Exception): pass
 
 @undoc
 class Bunch: pass
 
 
 def get_default_colors():
     if sys.platform=='darwin':
         return "LightBG"
     elif os.name=='nt':
         return 'Linux'
     else:
         return 'Linux'
 
 
 class SeparateUnicode(Unicode):
     """A Unicode subclass to validate separate_in, separate_out, etc.
 
     This is a Unicode based trait that converts '0'->'' and '\\n'->'\n'.
     """
 
     def validate(self, obj, value):
         if value == '0': value = ''
         value = value.replace('\\n','\n')
         return super(SeparateUnicode, self).validate(obj, value)
 
 
 class ReadlineNoRecord(object):
     """Context manager to execute some code, then reload readline history
     so that interactive input to the code doesn't appear when pressing up."""
     def __init__(self, shell):
         self.shell = shell
         self._nested_level = 0
 
     def __enter__(self):
         if self._nested_level == 0:
             try:
                 self.orig_length = self.current_length()
                 self.readline_tail = self.get_readline_tail()
             except (AttributeError, IndexError):   # Can fail with pyreadline
                 self.orig_length, self.readline_tail = 999999, []
         self._nested_level += 1
 
     def __exit__(self, type, value, traceback):
         self._nested_level -= 1
         if self._nested_level == 0:
             # Try clipping the end if it's got longer
             try:
                 e = self.current_length() - self.orig_length
                 if e > 0:
                     for _ in range(e):
                         self.shell.readline.remove_history_item(self.orig_length)
 
                 # If it still doesn't match, just reload readline history.
                 if self.current_length() != self.orig_length \
                     or self.get_readline_tail() != self.readline_tail:
                     self.shell.refill_readline_hist()
             except (AttributeError, IndexError):
                 pass
         # Returning False will cause exceptions to propagate
         return False
 
     def current_length(self):
         return self.shell.readline.get_current_history_length()
 
     def get_readline_tail(self, n=10):
         """Get the last n items in readline history."""
         end = self.shell.readline.get_current_history_length() + 1
         start = max(end-n, 1)
         ghi = self.shell.readline.get_history_item
         return [ghi(x) for x in range(start, end)]
 
 
 @undoc
 class DummyMod(object):
     """A dummy module used for IPython's interactive module when
     a namespace must be assigned to the module's __dict__."""
     pass
 
 #-----------------------------------------------------------------------------
 # Main IPython class
 #-----------------------------------------------------------------------------
 
 class InteractiveShell(SingletonConfigurable):
     """An enhanced, interactive shell for Python."""
 
     _instance = None
     
     ast_transformers = List([], config=True, help=
         """
         A list of ast.NodeTransformer subclass instances, which will be applied
         to user input before code is run.
         """
     )
 
     autocall = Enum((0,1,2), default_value=0, config=True, help=
         """
         Make IPython automatically call any callable object even if you didn't
         type explicit parentheses. For example, 'str 43' becomes 'str(43)'
         automatically. The value can be '0' to disable the feature, '1' for
         'smart' autocall, where it is not applied if there are no more
         arguments on the line, and '2' for 'full' autocall, where all callable
         objects are automatically called (even if no arguments are present).
         """
     )
     # TODO: remove all autoindent logic and put into frontends.
     # We can't do this yet because even runlines uses the autoindent.
     autoindent = CBool(True, config=True, help=
         """
         Autoindent IPython code entered interactively.
         """
     )
     automagic = CBool(True, config=True, help=
         """
         Enable magic commands to be called without the leading %.
         """
     )
     cache_size = Integer(1000, config=True, help=
         """
         Set the size of the output cache.  The default is 1000, you can
         change it permanently in your config file.  Setting it to 0 completely
         disables the caching system, and the minimum value accepted is 20 (if
         you provide a value less than 20, it is reset to 0 and a warning is
         issued).  This limit is defined because otherwise you'll spend more
         time re-flushing a too small cache than working
         """
     )
     color_info = CBool(True, config=True, help=
         """
         Use colors for displaying information about objects. Because this
         information is passed through a pager (like 'less'), and some pagers
         get confused with color codes, this capability can be turned off.
         """
     )
     colors = CaselessStrEnum(('NoColor','LightBG','Linux'),
                              default_value=get_default_colors(), config=True,
         help="Set the color scheme (NoColor, Linux, or LightBG)."
     )
     colors_force = CBool(False, help=
         """
         Force use of ANSI color codes, regardless of OS and readline
         availability.
         """
         # FIXME: This is essentially a hack to allow ZMQShell to show colors
         # without readline on Win32. When the ZMQ formatting system is
         # refactored, this should be removed.
     )
     debug = CBool(False, config=True)
     deep_reload = CBool(False, config=True, help=
         """
         Enable deep (recursive) reloading by default. IPython can use the
         deep_reload module which reloads changes in modules recursively (it
         replaces the reload() function, so you don't need to change anything to
         use it). deep_reload() forces a full reload of modules whose code may
         have changed, which the default reload() function does not.  When
         deep_reload is off, IPython will use the normal reload(), but
         deep_reload will still be available as dreload().
         """
     )
     disable_failing_post_execute = CBool(False, config=True,
         help="Don't call post-execute functions that have failed in the past."
     )
     display_formatter = Instance(DisplayFormatter)
     displayhook_class = Type(DisplayHook)
     display_pub_class = Type(DisplayPublisher)
     data_pub_class = None
 
     exit_now = CBool(False)
     exiter = Instance(ExitAutocall)
     def _exiter_default(self):
         return ExitAutocall(self)
     # Monotonically increasing execution counter
     execution_count = Integer(1)
     filename = Unicode("<ipython console>")
     ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
 
     # Input splitter, to transform input line by line and detect when a block
     # is ready to be executed.
     input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                               (), {'line_input_checker': True})
     
     # This InputSplitter instance is used to transform completed cells before
     # running them. It allows cell magics to contain blank lines.
     input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                                          (), {'line_input_checker': False})
     
     logstart = CBool(False, config=True, help=
         """
         Start logging to the default log file.
         """
     )
     logfile = Unicode('', config=True, help=
         """
         The name of the logfile to use.
         """
     )
     logappend = Unicode('', config=True, help=
         """
         Start logging to the given file in append mode.
         """
     )
     object_info_string_level = Enum((0,1,2), default_value=0,
                                     config=True)
     pdb = CBool(False, config=True, help=
         """
         Automatically call the pdb debugger after every exception.
         """
     )
     multiline_history = CBool(sys.platform != 'win32', config=True,
         help="Save multi-line entries as one entry in readline history"
     )
 
     # deprecated prompt traits:
     
     prompt_in1 = Unicode('In [\\#]: ', config=True,
         help="Deprecated, use PromptManager.in_template")
     prompt_in2 = Unicode('   .\\D.: ', config=True,
         help="Deprecated, use PromptManager.in2_template")
     prompt_out = Unicode('Out[\\#]: ', config=True,
         help="Deprecated, use PromptManager.out_template")
     prompts_pad_left = CBool(True, config=True,
         help="Deprecated, use PromptManager.justify")
     
     def _prompt_trait_changed(self, name, old, new):
         table = {
             'prompt_in1' : 'in_template',
             'prompt_in2' : 'in2_template',
             'prompt_out' : 'out_template',
             'prompts_pad_left' : 'justify',
         }
         warn("InteractiveShell.{name} is deprecated, use PromptManager.{newname}".format(
                 name=name, newname=table[name])
         )
         # protect against weird cases where self.config may not exist:
         if self.config is not None:
             # propagate to corresponding PromptManager trait
             setattr(self.config.PromptManager, table[name], new)
     
     _prompt_in1_changed = _prompt_trait_changed
     _prompt_in2_changed = _prompt_trait_changed
     _prompt_out_changed = _prompt_trait_changed
     _prompt_pad_left_changed = _prompt_trait_changed
     
     show_rewritten_input = CBool(True, config=True,
         help="Show rewritten input, e.g. for autocall."
     )
     
     quiet = CBool(False, config=True)
 
     history_length = Integer(10000, config=True)
 
     # The readline stuff will eventually be moved to the terminal subclass
     # but for now, we can't do that as readline is welded in everywhere.
     readline_use = CBool(True, config=True)
     readline_remove_delims = Unicode('-/~', config=True)
     readline_delims = Unicode() # set by init_readline()
     # don't use \M- bindings by default, because they
     # conflict with 8-bit encodings. See gh-58,gh-88
     readline_parse_and_bind = List([
             'tab: complete',
             '"\C-l": clear-screen',
             'set show-all-if-ambiguous on',
             '"\C-o": tab-insert',
             '"\C-r": reverse-search-history',
             '"\C-s": forward-search-history',
             '"\C-p": history-search-backward',
             '"\C-n": history-search-forward',
             '"\e[A": history-search-backward',
             '"\e[B": history-search-forward',
             '"\C-k": kill-line',
             '"\C-u": unix-line-discard',
         ], allow_none=False, config=True)
 
     ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none'],
                                   default_value='last_expr', config=True, 
                                   help="""
         'all', 'last', 'last_expr' or 'none', specifying which nodes should be
         run interactively (displaying output from expressions).""")
 
     # TODO: this part of prompt management should be moved to the frontends.
     # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
     separate_in = SeparateUnicode('\n', config=True)
     separate_out = SeparateUnicode('', config=True)
     separate_out2 = SeparateUnicode('', config=True)
     wildcards_case_sensitive = CBool(True, config=True)
     xmode = CaselessStrEnum(('Context','Plain', 'Verbose'),
                             default_value='Context', config=True)
 
     # Subcomponents of InteractiveShell
     alias_manager = Instance('IPython.core.alias.AliasManager')
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
     builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap')
     display_trap = Instance('IPython.core.display_trap.DisplayTrap')
     extension_manager = Instance('IPython.core.extensions.ExtensionManager')
     payload_manager = Instance('IPython.core.payload.PayloadManager')
     history_manager = Instance('IPython.core.history.HistoryManager')
     magics_manager = Instance('IPython.core.magic.MagicsManager')
 
     profile_dir = Instance('IPython.core.application.ProfileDir')
     @property
     def profile(self):
         if self.profile_dir is not None:
             name = os.path.basename(self.profile_dir.location)
             return name.replace('profile_','')
 
 
     # Private interface
     _post_execute = Instance(dict)
 
     # Tracks any GUI loop loaded for pylab
     pylab_gui_select = None
 
     def __init__(self, ipython_dir=None, profile_dir=None,
                  user_module=None, user_ns=None,
                  custom_exceptions=((), None), **kwargs):
 
         # This is where traits with a config_key argument are updated
         # from the values on config.
         super(InteractiveShell, self).__init__(**kwargs)
         self.configurables = [self]
 
         # These are relatively independent and stateless
         self.init_ipython_dir(ipython_dir)
         self.init_profile_dir(profile_dir)
         self.init_instance_attrs()
         self.init_environment()
         
         # Check if we're in a virtualenv, and set up sys.path.
         self.init_virtualenv()
 
         # Create namespaces (user_ns, user_global_ns, etc.)
         self.init_create_namespaces(user_module, user_ns)
         # This has to be done after init_create_namespaces because it uses
         # something in self.user_ns, but before init_sys_modules, which
         # is the first thing to modify sys.
         # TODO: When we override sys.stdout and sys.stderr before this class
         # is created, we are saving the overridden ones here. Not sure if this
         # is what we want to do.
         self.save_sys_module_state()
         self.init_sys_modules()
 
         # While we're trying to have each part of the code directly access what
         # it needs without keeping redundant references to objects, we have too
         # much legacy code that expects ip.db to exist.
         self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
 
         self.init_history()
         self.init_encoding()
         self.init_prefilter()
 
         self.init_syntax_highlighting()
         self.init_hooks()
         self.init_pushd_popd_magic()
         # self.init_traceback_handlers use to be here, but we moved it below
         # because it and init_io have to come after init_readline.
         self.init_user_ns()
         self.init_logger()
         self.init_builtins()
 
         # The following was in post_config_initialization
         self.init_inspector()
         # init_readline() must come before init_io(), because init_io uses
         # readline related things.
         self.init_readline()
         # We save this here in case user code replaces raw_input, but it needs
         # to be after init_readline(), because PyPy's readline works by replacing
         # raw_input.
         if py3compat.PY3:
             self.raw_input_original = input
         else:
             self.raw_input_original = raw_input
         # init_completer must come after init_readline, because it needs to
         # know whether readline is present or not system-wide to configure the
         # completers, since the completion machinery can now operate
         # independently of readline (e.g. over the network)
         self.init_completer()
         # TODO: init_io() needs to happen before init_traceback handlers
         # because the traceback handlers hardcode the stdout/stderr streams.
         # This logic in in debugger.Pdb and should eventually be changed.
         self.init_io()
         self.init_traceback_handlers(custom_exceptions)
         self.init_prompts()
         self.init_display_formatter()
         self.init_display_pub()
         self.init_data_pub()
         self.init_displayhook()
         self.init_latextool()
         self.init_magics()
         self.init_alias()
         self.init_logstart()
         self.init_pdb()
         self.init_extension_manager()
         self.init_payload()
         self.init_comms()
         self.hooks.late_startup_hook()
         atexit.register(self.atexit_operations)
 
     def get_ipython(self):
         """Return the currently running IPython instance."""
         return self
 
     #-------------------------------------------------------------------------
     # Trait changed handlers
     #-------------------------------------------------------------------------
 
     def _ipython_dir_changed(self, name, new):
         if not os.path.isdir(new):
             os.makedirs(new, mode = 0o777)
 
     def set_autoindent(self,value=None):
         """Set the autoindent flag, checking for readline support.
 
         If called with no arguments, it acts as a toggle."""
 
         if value != 0 and not self.has_readline:
             if os.name == 'posix':
                 warn("The auto-indent feature requires the readline library")
             self.autoindent = 0
             return
         if value is None:
             self.autoindent = not self.autoindent
         else:
             self.autoindent = value
 
     #-------------------------------------------------------------------------
     # init_* methods called by __init__
     #-------------------------------------------------------------------------
 
     def init_ipython_dir(self, ipython_dir):
         if ipython_dir is not None:
             self.ipython_dir = ipython_dir
             return
 
         self.ipython_dir = get_ipython_dir()
 
     def init_profile_dir(self, profile_dir):
         if profile_dir is not None:
             self.profile_dir = profile_dir
             return
         self.profile_dir =\
             ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
 
     def init_instance_attrs(self):
         self.more = False
 
         # command compiler
         self.compile = CachingCompiler()
 
         # Make an empty namespace, which extension writers can rely on both
         # existing and NEVER being used by ipython itself.  This gives them a
         # convenient location for storing additional information and state
         # their extensions may require, without fear of collisions with other
         # ipython names that may develop later.
         self.meta = Struct()
 
         # Temporary files used for various purposes.  Deleted at exit.
         self.tempfiles = []
 
         # Keep track of readline usage (later set by init_readline)
         self.has_readline = False
 
         # keep track of where we started running (mainly for crash post-mortem)
         # This is not being used anywhere currently.
-        self.starting_dir = os.getcwdu()
+        self.starting_dir = py3compat.getcwd()
 
         # Indentation management
         self.indent_current_nsp = 0
 
         # Dict to track post-execution functions that have been registered
         self._post_execute = {}
 
     def init_environment(self):
         """Any changes we need to make to the user's environment."""
         pass
 
     def init_encoding(self):
         # Get system encoding at startup time.  Certain terminals (like Emacs
         # under Win32 have it set to None, and we need to have a known valid
         # encoding to use in the raw_input() method
         try:
             self.stdin_encoding = sys.stdin.encoding or 'ascii'
         except AttributeError:
             self.stdin_encoding = 'ascii'
 
     def init_syntax_highlighting(self):
         # Python source parser/formatter for syntax highlighting
         pyformat = PyColorize.Parser().format
         self.pycolorize = lambda src: pyformat(src,'str',self.colors)
 
     def init_pushd_popd_magic(self):
         # for pushd/popd management
         self.home_dir = get_home_dir()
 
         self.dir_stack = []
 
     def init_logger(self):
         self.logger = Logger(self.home_dir, logfname='ipython_log.py',
                              logmode='rotate')
 
     def init_logstart(self):
         """Initialize logging in case it was requested at the command line.
         """
         if self.logappend:
             self.magic('logstart %s append' % self.logappend)
         elif self.logfile:
             self.magic('logstart %s' % self.logfile)
         elif self.logstart:
             self.magic('logstart')
 
     def init_builtins(self):
         # A single, static flag that we set to True.  Its presence indicates
         # that an IPython shell has been created, and we make no attempts at
         # removing on exit or representing the existence of more than one
         # IPython at a time.
         builtin_mod.__dict__['__IPYTHON__'] = True
 
         # In 0.11 we introduced '__IPYTHON__active' as an integer we'd try to
         # manage on enter/exit, but with all our shells it's virtually
         # impossible to get all the cases right.  We're leaving the name in for
         # those who adapted their codes to check for this flag, but will
         # eventually remove it after a few more releases.
         builtin_mod.__dict__['__IPYTHON__active'] = \
                                           'Deprecated, check for __IPYTHON__'
 
         self.builtin_trap = BuiltinTrap(shell=self)
 
     def init_inspector(self):
         # Object inspector
         self.inspector = oinspect.Inspector(oinspect.InspectColors,
                                             PyColorize.ANSICodeColors,
                                             'NoColor',
                                             self.object_info_string_level)
 
     def init_io(self):
         # This will just use sys.stdout and sys.stderr. If you want to
         # override sys.stdout and sys.stderr themselves, you need to do that
         # *before* instantiating this class, because io holds onto
         # references to the underlying streams.
         if (sys.platform == 'win32' or sys.platform == 'cli') and self.has_readline:
             io.stdout = io.stderr = io.IOStream(self.readline._outputfile)
         else:
             io.stdout = io.IOStream(sys.stdout)
             io.stderr = io.IOStream(sys.stderr)
 
     def init_prompts(self):
         self.prompt_manager = PromptManager(shell=self, parent=self)
         self.configurables.append(self.prompt_manager)
         # Set system prompts, so that scripts can decide if they are running
         # interactively.
         sys.ps1 = 'In : '
         sys.ps2 = '...: '
         sys.ps3 = 'Out: '
 
     def init_display_formatter(self):
         self.display_formatter = DisplayFormatter(parent=self)
         self.configurables.append(self.display_formatter)
 
     def init_display_pub(self):
         self.display_pub = self.display_pub_class(parent=self)
         self.configurables.append(self.display_pub)
 
     def init_data_pub(self):
         if not self.data_pub_class:
             self.data_pub = None
             return
         self.data_pub = self.data_pub_class(parent=self)
         self.configurables.append(self.data_pub)
 
     def init_displayhook(self):
         # Initialize displayhook, set in/out prompts and printing system
         self.displayhook = self.displayhook_class(
             parent=self,
             shell=self,
             cache_size=self.cache_size,
         )
         self.configurables.append(self.displayhook)
         # This is a context manager that installs/revmoes the displayhook at
         # the appropriate time.
         self.display_trap = DisplayTrap(hook=self.displayhook)
 
     def init_latextool(self):
         """Configure LaTeXTool."""
         cfg = LaTeXTool.instance(parent=self)
         if cfg not in self.configurables:
             self.configurables.append(cfg)
 
     def init_virtualenv(self):
         """Add a virtualenv to sys.path so the user can import modules from it.
         This isn't perfect: it doesn't use the Python interpreter with which the
         virtualenv was built, and it ignores the --no-site-packages option. A
         warning will appear suggesting the user installs IPython in the
         virtualenv, but for many cases, it probably works well enough.
         
         Adapted from code snippets online.
         
         http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
         """
         if 'VIRTUAL_ENV' not in os.environ:
             # Not in a virtualenv
             return
         
         if sys.executable.startswith(os.environ['VIRTUAL_ENV']):
             # Running properly in the virtualenv, don't need to do anything
             return
         
         warn("Attempting to work in a virtualenv. If you encounter problems, please "
              "install IPython inside the virtualenv.")
         if sys.platform == "win32":
             virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages') 
         else:
             virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',
                        'python%d.%d' % sys.version_info[:2], 'site-packages')
         
         import site
         sys.path.insert(0, virtual_env)
         site.addsitedir(virtual_env)
 
     #-------------------------------------------------------------------------
     # Things related to injections into the sys module
     #-------------------------------------------------------------------------
 
     def save_sys_module_state(self):
         """Save the state of hooks in the sys module.
 
         This has to be called after self.user_module is created.
         """
         self._orig_sys_module_state = {}
         self._orig_sys_module_state['stdin'] = sys.stdin
         self._orig_sys_module_state['stdout'] = sys.stdout
         self._orig_sys_module_state['stderr'] = sys.stderr
         self._orig_sys_module_state['excepthook'] = sys.excepthook
         self._orig_sys_modules_main_name = self.user_module.__name__
         self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
 
     def restore_sys_module_state(self):
         """Restore the state of the sys module."""
         try:
             for k, v in iteritems(self._orig_sys_module_state):
                 setattr(sys, k, v)
         except AttributeError:
             pass
         # Reset what what done in self.init_sys_modules
         if self._orig_sys_modules_main_mod is not None:
             sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
 
     #-------------------------------------------------------------------------
     # Things related to hooks
     #-------------------------------------------------------------------------
 
     def init_hooks(self):
         # hooks holds pointers used for user-side customizations
         self.hooks = Struct()
 
         self.strdispatchers = {}
 
         # Set all default hooks, defined in the IPython.hooks module.
         hooks = IPython.core.hooks
         for hook_name in hooks.__all__:
             # default hooks have priority 100, i.e. low; user hooks should have
             # 0-100 priority
             self.set_hook(hook_name,getattr(hooks,hook_name), 100)
 
     def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
         """set_hook(name,hook) -> sets an internal IPython hook.
 
         IPython exposes some of its internal API as user-modifiable hooks.  By
         adding your function to one of these hooks, you can modify IPython's
         behavior to call at runtime your own routines."""
 
         # At some point in the future, this should validate the hook before it
         # accepts it.  Probably at least check that the hook takes the number
         # of args it's supposed to.
 
         f = types.MethodType(hook,self)
 
         # check if the hook is for strdispatcher first
         if str_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_s(str_key, f, priority )
             self.strdispatchers[name] = sdp
             return
         if re_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_re(re.compile(re_key), f, priority )
             self.strdispatchers[name] = sdp
             return
 
         dp = getattr(self.hooks, name, None)
         if name not in IPython.core.hooks.__all__:
             print("Warning! Hook '%s' is not one of %s" % \
                   (name, IPython.core.hooks.__all__ ))
         if not dp:
             dp = IPython.core.hooks.CommandChainDispatcher()
 
         try:
             dp.add(f,priority)
         except AttributeError:
             # it was not commandchain, plain old func - replace
             dp = f
 
         setattr(self.hooks,name, dp)
 
     def register_post_execute(self, func):
         """Register a function for calling after code execution.
         """
         if not callable(func):
             raise ValueError('argument %s must be callable' % func)
         self._post_execute[func] = True
     
     #-------------------------------------------------------------------------
     # Things related to the "main" module
     #-------------------------------------------------------------------------
 
     def new_main_mod(self, filename, modname):
         """Return a new 'main' module object for user code execution.
         
         ``filename`` should be the path of the script which will be run in the
         module. Requests with the same filename will get the same module, with
         its namespace cleared.
         
         ``modname`` should be the module name - normally either '__main__' or
         the basename of the file without the extension.
         
         When scripts are executed via %run, we must keep a reference to their
         __main__ module around so that Python doesn't
         clear it, rendering references to module globals useless.
 
         This method keeps said reference in a private dict, keyed by the
         absolute path of the script. This way, for multiple executions of the
         same script we only keep one copy of the namespace (the last one),
         thus preventing memory leaks from old references while allowing the
         objects from the last execution to be accessible.
         """
         filename = os.path.abspath(filename)
         try:
             main_mod = self._main_mod_cache[filename]
         except KeyError:
             main_mod = self._main_mod_cache[filename] = types.ModuleType(modname,
                         doc="Module created for script run in IPython")
         else:
             main_mod.__dict__.clear()
             main_mod.__name__ = modname
         
         main_mod.__file__ = filename
         # It seems pydoc (and perhaps others) needs any module instance to
         # implement a __nonzero__ method
         main_mod.__nonzero__ = lambda : True
         
         return main_mod
 
     def clear_main_mod_cache(self):
         """Clear the cache of main modules.
 
         Mainly for use by utilities like %reset.
 
         Examples
         --------
 
         In [15]: import IPython
 
         In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
 
         In [17]: len(_ip._main_mod_cache) > 0
         Out[17]: True
 
         In [18]: _ip.clear_main_mod_cache()
 
         In [19]: len(_ip._main_mod_cache) == 0
         Out[19]: True
         """
         self._main_mod_cache.clear()
 
     #-------------------------------------------------------------------------
     # Things related to debugging
     #-------------------------------------------------------------------------
 
     def init_pdb(self):
         # Set calling of pdb on exceptions
         # self.call_pdb is a property
         self.call_pdb = self.pdb
 
     def _get_call_pdb(self):
         return self._call_pdb
 
     def _set_call_pdb(self,val):
 
         if val not in (0,1,False,True):
             raise ValueError('new call_pdb value must be boolean')
 
         # store value in instance
         self._call_pdb = val
 
         # notify the actual exception handlers
         self.InteractiveTB.call_pdb = val
 
     call_pdb = property(_get_call_pdb,_set_call_pdb,None,
                         'Control auto-activation of pdb at exceptions')
 
     def debugger(self,force=False):
         """Call the pydb/pdb debugger.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
             flag and does not actually invoke the debugger if the flag is false.
             The 'force' option forces the debugger to activate even if the flag
             is false.
         """
 
         if not (force or self.call_pdb):
             return
 
         if not hasattr(sys,'last_traceback'):
             error('No traceback has been produced, nothing to debug.')
             return
 
         # use pydb if available
         if debugger.has_pydb:
             from pydb import pm
         else:
             # fallback to our internal debugger
             pm = lambda : self.InteractiveTB.debugger(force=True)
 
         with self.readline_no_record:
             pm()
 
     #-------------------------------------------------------------------------
     # Things related to IPython's various namespaces
     #-------------------------------------------------------------------------
     default_user_namespaces = True
 
     def init_create_namespaces(self, user_module=None, user_ns=None):
         # Create the namespace where the user will operate.  user_ns is
         # normally the only one used, and it is passed to the exec calls as
         # the locals argument.  But we do carry a user_global_ns namespace
         # given as the exec 'globals' argument,  This is useful in embedding
         # situations where the ipython shell opens in a context where the
         # distinction between locals and globals is meaningful.  For
         # non-embedded contexts, it is just the same object as the user_ns dict.
 
         # FIXME. For some strange reason, __builtins__ is showing up at user
         # level as a dict instead of a module. This is a manual fix, but I
         # should really track down where the problem is coming from. Alex
         # Schmolck reported this problem first.
 
         # A useful post by Alex Martelli on this topic:
         # Re: inconsistent value from __builtins__
         # Von: Alex Martelli <[e-mail]>
         # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
         # Gruppen: comp.lang.python
 
         # Michael Hohn <[e-mail]> wrote:
         # > >>> print type(builtin_check.get_global_binding('__builtins__'))
         # > <type 'dict'>
         # > >>> print type(__builtins__)
         # > <type 'module'>
         # > Is this difference in return value intentional?
 
         # Well, it's documented that '__builtins__' can be either a dictionary
         # or a module, and it's been that way for a long time. Whether it's
         # intentional (or sensible), I don't know. In any case, the idea is
         # that if you need to access the built-in namespace directly, you
         # should start with "import __builtin__" (note, no 's') which will
         # definitely give you a module. Yeah, it's somewhat confusing:-(.
 
         # These routines return a properly built module and dict as needed by
         # the rest of the code, and can also be used by extension writers to
         # generate properly initialized namespaces.
         if (user_ns is not None) or (user_module is not None):
             self.default_user_namespaces = False
         self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
 
         # A record of hidden variables we have added to the user namespace, so
         # we can list later only variables defined in actual interactive use.
         self.user_ns_hidden = {}
 
         # Now that FakeModule produces a real module, we've run into a nasty
         # problem: after script execution (via %run), the module where the user
         # code ran is deleted.  Now that this object is a true module (needed
         # so docetst and other tools work correctly), the Python module
         # teardown mechanism runs over it, and sets to None every variable
         # present in that module.  Top-level references to objects from the
         # script survive, because the user_ns is updated with them.  However,
         # calling functions defined in the script that use other things from
         # the script will fail, because the function's closure had references
         # to the original objects, which are now all None.  So we must protect
         # these modules from deletion by keeping a cache.
         #
         # To avoid keeping stale modules around (we only need the one from the
         # last run), we use a dict keyed with the full path to the script, so
         # only the last version of the module is held in the cache.  Note,
         # however, that we must cache the module *namespace contents* (their
         # __dict__).  Because if we try to cache the actual modules, old ones
         # (uncached) could be destroyed while still holding references (such as
         # those held by GUI objects that tend to be long-lived)>
         #
         # The %reset command will flush this cache.  See the cache_main_mod()
         # and clear_main_mod_cache() methods for details on use.
 
         # This is the cache used for 'main' namespaces
         self._main_mod_cache = {}
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
         self.ns_table = {'user_global':self.user_module.__dict__,
                          'user_local':self.user_ns,
                          'builtin':builtin_mod.__dict__
                          }
     
     @property
     def user_global_ns(self):
         return self.user_module.__dict__
 
     def prepare_user_module(self, user_module=None, user_ns=None):
         """Prepare the module and namespace in which user code will be run.
         
         When IPython is started normally, both parameters are None: a new module
         is created automatically, and its __dict__ used as the namespace.
         
         If only user_module is provided, its __dict__ is used as the namespace.
         If only user_ns is provided, a dummy module is created, and user_ns
         becomes the global namespace. If both are provided (as they may be
         when embedding), user_ns is the local namespace, and user_module
         provides the global namespace.
 
         Parameters
         ----------
         user_module : module, optional
             The current user module in which IPython is being run. If None,
             a clean module will be created.
         user_ns : dict, optional
             A namespace in which to run interactive commands.
 
         Returns
         -------
         A tuple of user_module and user_ns, each properly initialised.
         """
         if user_module is None and user_ns is not None:
             user_ns.setdefault("__name__", "__main__")
             user_module = DummyMod()
             user_module.__dict__ = user_ns
             
         if user_module is None:
             user_module = types.ModuleType("__main__",
                 doc="Automatically created module for IPython interactive environment")
         
         # We must ensure that __builtin__ (without the final 's') is always
         # available and pointing to the __builtin__ *module*.  For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         user_module.__dict__.setdefault('__builtin__', builtin_mod)
         user_module.__dict__.setdefault('__builtins__', builtin_mod)
         
         if user_ns is None:
             user_ns = user_module.__dict__
 
         return user_module, user_ns
 
     def init_sys_modules(self):
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
         # everything out of __main__, but for embedding purposes each IPython
         # instance has its own private namespace, so we can't go shoving
         # everything into __main__.
 
         # note, however, that we should only do this for non-embedded
         # ipythons, which really mimic the __main__.__dict__ with their own
         # namespace.  Embedded instances, on the other hand, should not do
         # this because they need to manage the user local/global namespaces
         # only, but they live within a 'normal' __main__ (meaning, they
         # shouldn't overtake the execution environment of the script they're
         # embedded in).
 
         # This is overridden in the InteractiveShellEmbed subclass to a no-op.
         main_name = self.user_module.__name__
         sys.modules[main_name] = self.user_module
 
     def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
 
         Certain history lists are also initialized here, as they effectively
         act as user namespaces.
 
         Notes
         -----
         All data structures here are only filled in, they are NOT reset by this
         method.  If they were not empty before, data will simply be added to
         therm.
         """
         # This function works in two parts: first we put a few things in
         # user_ns, and we sync that contents into user_ns_hidden so that these
         # initial variables aren't shown by %who.  After the sync, we add the
         # rest of what we *do* want the user to see with %who even on a new
         # session (probably nothing, so theye really only see their own stuff)
 
         # The user dict must *always* have a __builtin__ reference to the
         # Python standard __builtin__ namespace,  which must be imported.
         # This is so that certain operations in prompt evaluation can be
         # reliably executed with builtins.  Note that we can NOT use
         # __builtins__ (note the 's'),  because that can either be a dict or a
         # module, and can even mutate at runtime, depending on the context
         # (Python makes no guarantees on it).  In contrast, __builtin__ is
         # always a module object, though it must be explicitly imported.
 
         # For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         ns = dict()
         
         # Put 'help' in the user namespace
         try:
             from site import _Helper
             ns['help'] = _Helper()
         except ImportError:
             warn('help() not available - check site.py')
 
         # make global variables for user access to the histories
         ns['_ih'] = self.history_manager.input_hist_parsed
         ns['_oh'] = self.history_manager.output_hist
         ns['_dh'] = self.history_manager.dir_hist
 
         ns['_sh'] = shadowns
 
         # user aliases to input and output histories.  These shouldn't show up
         # in %who, as they can have very large reprs.
         ns['In']  = self.history_manager.input_hist_parsed
         ns['Out'] = self.history_manager.output_hist
 
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
         
         ns['exit'] = self.exiter
         ns['quit'] = self.exiter
 
         # Sync what we've added so far to user_ns_hidden so these aren't seen
         # by %who
         self.user_ns_hidden.update(ns)
 
         # Anything put into ns now would show up in %who.  Think twice before
         # putting anything here, as we really want %who to show the user their
         # stuff, not our variables.
 
         # Finally, update the real user's namespace
         self.user_ns.update(ns)
     
     @property
     def all_ns_refs(self):
         """Get a list of references to all the namespace dictionaries in which
         IPython might store a user-created object.
         
         Note that this does not include the displayhook, which also caches
         objects from the output."""
         return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
                [m.__dict__ for m in self._main_mod_cache.values()]
 
     def reset(self, new_session=True):
         """Clear all internal namespaces, and attempt to release references to
         user objects.
 
         If new_session is True, a new history session will be opened.
         """
         # Clear histories
         self.history_manager.reset(new_session)
         # Reset counter used to index all histories
         if new_session:
             self.execution_count = 1
 
         # Flush cached output items
         if self.displayhook.do_full_cache:
             self.displayhook.flush()
 
         # The main execution namespaces must be cleared very carefully,
         # skipping the deletion of the builtin-related keys, because doing so
         # would cause errors in many object's __del__ methods.
         if self.user_ns is not self.user_global_ns:
             self.user_ns.clear()
         ns = self.user_global_ns
         drop_keys = set(ns.keys())
         drop_keys.discard('__builtin__')
         drop_keys.discard('__builtins__')
         drop_keys.discard('__name__')
         for k in drop_keys:
             del ns[k]
         
         self.user_ns_hidden.clear()
         
         # Restore the user namespaces to minimal usability
         self.init_user_ns()
 
         # Restore the default and user aliases
         self.alias_manager.clear_aliases()
         self.alias_manager.init_aliases()
 
         # Flush the private list of module references kept for script
         # execution protection
         self.clear_main_mod_cache()
 
     def del_var(self, varname, by_name=False):
         """Delete a variable from the various namespaces, so that, as
         far as possible, we're not keeping any hidden references to it.
 
         Parameters
         ----------
         varname : str
             The name of the variable to delete.
         by_name : bool
             If True, delete variables with the given name in each
             namespace. If False (default), find the variable in the user
             namespace, and delete references to it.
         """
         if varname in ('__builtin__', '__builtins__'):
             raise ValueError("Refusing to delete %s" % varname)
 
         ns_refs = self.all_ns_refs
         
         if by_name:                    # Delete by name
             for ns in ns_refs:
                 try:
                     del ns[varname]
                 except KeyError:
                     pass
         else:                         # Delete by object
             try:
                 obj = self.user_ns[varname]
             except KeyError:
                 raise NameError("name '%s' is not defined" % varname)
             # Also check in output history
             ns_refs.append(self.history_manager.output_hist)
             for ns in ns_refs:
                 to_delete = [n for n, o in iteritems(ns) if o is obj]
                 for name in to_delete:
                     del ns[name]
 
             # displayhook keeps extra references, but not in a dictionary
             for name in ('_', '__', '___'):
                 if getattr(self.displayhook, name) is obj:
                     setattr(self.displayhook, name, None)
 
     def reset_selective(self, regex=None):
         """Clear selective variables from internal namespaces based on a
         specified regular expression.
 
         Parameters
         ----------
         regex : string or compiled pattern, optional
             A regular expression pattern that will be used in searching
             variable names in the users namespaces.
         """
         if regex is not None:
             try:
                 m = re.compile(regex)
             except TypeError:
                 raise TypeError('regex must be a string or compiled pattern')
             # Search for keys in each namespace that match the given regex
             # If a match is found, delete the key/value pair.
             for ns in self.all_ns_refs:
                 for var in ns:
                     if m.search(var):
                         del ns[var]
 
     def push(self, variables, interactive=True):
         """Inject a group of variables into the IPython user namespace.
 
         Parameters
         ----------
         variables : dict, str or list/tuple of str
             The variables to inject into the user's namespace.  If a dict, a
             simple update is done.  If a str, the string is assumed to have
             variable names separated by spaces.  A list/tuple of str can also
             be used to give the variable names.  If just the variable names are
             give (list/tuple/str) then the variable values looked up in the
             callers frame.
         interactive : bool
             If True (default), the variables will be listed with the ``who``
             magic.
         """
         vdict = None
 
         # We need a dict of name/value pairs to do namespace updates.
         if isinstance(variables, dict):
             vdict = variables
         elif isinstance(variables, string_types+(list, tuple)):
             if isinstance(variables, string_types):
                 vlist = variables.split()
             else:
                 vlist = variables
             vdict = {}
             cf = sys._getframe(1)
             for name in vlist:
                 try:
                     vdict[name] = eval(name, cf.f_globals, cf.f_locals)
                 except:
                     print('Could not get variable %s from %s' %
                            (name,cf.f_code.co_name))
         else:
             raise ValueError('variables must be a dict/str/list/tuple')
 
         # Propagate variables to user namespace
         self.user_ns.update(vdict)
 
         # And configure interactive visibility
         user_ns_hidden = self.user_ns_hidden
         if interactive:
             for name in vdict:
                 user_ns_hidden.pop(name, None)
         else:
             user_ns_hidden.update(vdict)
     
     def drop_by_id(self, variables):
         """Remove a dict of variables from the user namespace, if they are the
         same as the values in the dictionary.
         
         This is intended for use by extensions: variables that they've added can
         be taken back out if they are unloaded, without removing any that the
         user has overwritten.
         
         Parameters
         ----------
         variables : dict
           A dictionary mapping object names (as strings) to the objects.
         """
         for name, obj in iteritems(variables):
             if name in self.user_ns and self.user_ns[name] is obj:
                 del self.user_ns[name]
                 self.user_ns_hidden.pop(name, None)
 
     #-------------------------------------------------------------------------
     # Things related to object introspection
     #-------------------------------------------------------------------------
 
     def _ofind(self, oname, namespaces=None):
         """Find an object in the available namespaces.
 
         self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
 
         Has special code to detect magic functions.
         """
         oname = oname.strip()
         #print '1- oname: <%r>' % oname  # dbg
         if not oname.startswith(ESC_MAGIC) and \
             not oname.startswith(ESC_MAGIC2) and \
             not py3compat.isidentifier(oname, dotted=True):
             return dict(found=False)
 
         alias_ns = None
         if namespaces is None:
             # Namespaces to search in:
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.user_ns),
                            ('Interactive (global)', self.user_global_ns),
                            ('Python builtin', builtin_mod.__dict__),
                            ]
 
         # initialize results to 'null'
         found = False; obj = None;  ospace = None;  ds = None;
         ismagic = False; isalias = False; parent = None
 
         # We need to special-case 'print', which as of python2.6 registers as a
         # function but should only be treated as one if print_function was
         # loaded with a future import.  In this case, just bail.
         if (oname == 'print' and not py3compat.PY3 and not \
             (self.compile.compiler_flags & __future__.CO_FUTURE_PRINT_FUNCTION)):
             return {'found':found, 'obj':obj, 'namespace':ospace,
                     'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
 
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
         oname_parts = oname.split('.')
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
                 obj = ns[oname_head]
             except KeyError:
                 continue
             else:
                 #print 'oname_rest:', oname_rest  # dbg
                 for part in oname_rest:
                     try:
                         parent = obj
                         obj = getattr(obj,part)
                     except:
                         # Blanket except b/c some badly implemented objects
                         # allow __getattr__ to raise exceptions other than
                         # AttributeError, which then crashes IPython.
                         break
                 else:
                     # If we finish the for loop (no break), we got all members
                     found = True
                     ospace = nsname
                     break  # namespace loop
 
         # Try to see if it's magic
         if not found:
             obj = None
             if oname.startswith(ESC_MAGIC2):
                 oname = oname.lstrip(ESC_MAGIC2)
                 obj = self.find_cell_magic(oname)
             elif oname.startswith(ESC_MAGIC):
                 oname = oname.lstrip(ESC_MAGIC)
                 obj = self.find_line_magic(oname)
             else:
                 # search without prefix, so run? will find %run?
                 obj = self.find_line_magic(oname)
                 if obj is None:
                     obj = self.find_cell_magic(oname)
             if obj is not None:
                 found = True
                 ospace = 'IPython internal'
                 ismagic = True
 
         # Last try: special-case some literals like '', [], {}, etc:
         if not found and oname_head in ["''",'""','[]','{}','()']:
             obj = eval(oname_head)
             found = True
             ospace = 'Interactive'
 
         return {'found':found, 'obj':obj, 'namespace':ospace,
                 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
 
     def _ofind_property(self, oname, info):
         """Second part of object finding, to look for property details."""
         if info.found:
             # Get the docstring of the class property if it exists.
             path = oname.split('.')
             root = '.'.join(path[:-1])
             if info.parent is not None:
                 try:
                     target = getattr(info.parent, '__class__')
                     # The object belongs to a class instance.
                     try:
                         target = getattr(target, path[-1])
                         # The class defines the object.
                         if isinstance(target, property):
                             oname = root + '.__class__.' + path[-1]
                             info = Struct(self._ofind(oname))
                     except AttributeError: pass
                 except AttributeError: pass
 
         # We return either the new info or the unmodified input if the object
         # hadn't been found
         return info
 
     def _object_find(self, oname, namespaces=None):
         """Find an object and return a struct with info about it."""
         inf = Struct(self._ofind(oname, namespaces))
         return Struct(self._ofind_property(oname, inf))
 
     def _inspect(self, meth, oname, namespaces=None, **kw):
         """Generic interface to the inspector system.
 
         This function is meant to be called by pdef, pdoc & friends."""
         info = self._object_find(oname, namespaces)
         if info.found:
             pmethod = getattr(self.inspector, meth)
             formatter = format_screen if info.ismagic else None
             if meth == 'pdoc':
                 pmethod(info.obj, oname, formatter)
             elif meth == 'pinfo':
                 pmethod(info.obj, oname, formatter, info, **kw)
             else:
                 pmethod(info.obj, oname)
         else:
             print('Object `%s` not found.' % oname)
             return 'not found'  # so callers can take other action
 
     def object_inspect(self, oname, detail_level=0):
         with self.builtin_trap:
             info = self._object_find(oname)
             if info.found:
                 return self.inspector.info(info.obj, oname, info=info,
                             detail_level=detail_level
                 )
             else:
                 return oinspect.object_info(name=oname, found=False)
 
     #-------------------------------------------------------------------------
     # Things related to history management
     #-------------------------------------------------------------------------
 
     def init_history(self):
         """Sets up the command history, and starts regular autosaves."""
         self.history_manager = HistoryManager(shell=self, parent=self)
         self.configurables.append(self.history_manager)
 
     #-------------------------------------------------------------------------
     # Things related to exception handling and tracebacks (not debugging)
     #-------------------------------------------------------------------------
 
     def init_traceback_handlers(self, custom_exceptions):
         # Syntax error handler.
         self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor')
 
         # The interactive one is initialized with an offset, meaning we always
         # want to remove the topmost item in the traceback, which is our own
         # internal code. Valid modes: ['Plain','Context','Verbose']
         self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
                                                      tb_offset = 1,
                                    check_cache=check_linecache_ipython)
 
         # The instance will store a pointer to the system-wide exception hook,
         # so that runtime code (such as magics) can access it.  This is because
         # during the read-eval loop, it may get temporarily overwritten.
         self.sys_excepthook = sys.excepthook
 
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
         # Set the exception mode
         self.InteractiveTB.set_mode(mode=self.xmode)
 
     def set_custom_exc(self, exc_tuple, handler):
         """set_custom_exc(exc_tuple,handler)
 
         Set a custom exception handler, which will be called if any of the
         exceptions in exc_tuple occur in the mainloop (specifically, in the
         run_code() method).
 
         Parameters
         ----------
 
         exc_tuple : tuple of exception classes
             A *tuple* of exception classes, for which to call the defined
             handler.  It is very important that you use a tuple, and NOT A
             LIST here, because of the way Python's except statement works.  If
             you only want to trap a single exception, use a singleton tuple::
 
                 exc_tuple == (MyCustomException,)
 
         handler : callable
             handler must have the following signature::
 
                 def my_handler(self, etype, value, tb, tb_offset=None):
                     ...
                     return structured_traceback
 
             Your handler must return a structured traceback (a list of strings),
             or None.
 
             This will be made into an instance method (via types.MethodType)
             of IPython itself, and it will be called if any of the exceptions
             listed in the exc_tuple are caught. If the handler is None, an
             internal basic one is used, which just prints basic info.
 
             To protect IPython from crashes, if your handler ever raises an
             exception or returns an invalid result, it will be immediately
             disabled.
 
         WARNING: by putting in your own exception handler into IPython's main
         execution loop, you run a very good chance of nasty crashes.  This
         facility should only be used if you really know what you are doing."""
 
         assert type(exc_tuple)==type(()) , \
                "The custom exceptions must be given AS A TUPLE."
 
         def dummy_handler(self,etype,value,tb,tb_offset=None):
             print('*** Simple custom exception handler ***')
             print('Exception type :',etype)
             print('Exception value:',value)
             print('Traceback      :',tb)
             #print 'Source code    :','\n'.join(self.buffer)
         
         def validate_stb(stb):
             """validate structured traceback return type
             
             return type of CustomTB *should* be a list of strings, but allow
             single strings or None, which are harmless.
             
             This function will *always* return a list of strings,
             and will raise a TypeError if stb is inappropriate.
             """
             msg = "CustomTB must return list of strings, not %r" % stb
             if stb is None:
                 return []
             elif isinstance(stb, string_types):
                 return [stb]
             elif not isinstance(stb, list):
                 raise TypeError(msg)
             # it's a list
             for line in stb:
                 # check every element
                 if not isinstance(line, string_types):
                     raise TypeError(msg)
             return stb
 
         if handler is None:
             wrapped = dummy_handler
         else:
             def wrapped(self,etype,value,tb,tb_offset=None):
                 """wrap CustomTB handler, to protect IPython from user code
                 
                 This makes it harder (but not impossible) for custom exception
                 handlers to crash IPython.
                 """
                 try:
                     stb = handler(self,etype,value,tb,tb_offset=tb_offset)
                     return validate_stb(stb)
                 except:
                     # clear custom handler immediately
                     self.set_custom_exc((), None)
                     print("Custom TB Handler failed, unregistering", file=io.stderr)
                     # show the exception in handler first
                     stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
                     print(self.InteractiveTB.stb2text(stb), file=io.stdout)
                     print("The original exception:", file=io.stdout)
                     stb = self.InteractiveTB.structured_traceback(
                                             (etype,value,tb), tb_offset=tb_offset
                     )
                 return stb
 
         self.CustomTB = types.MethodType(wrapped,self)
         self.custom_exceptions = exc_tuple
 
     def excepthook(self, etype, value, tb):
       """One more defense for GUI apps that call sys.excepthook.
 
       GUI frameworks like wxPython trap exceptions and call
       sys.excepthook themselves.  I guess this is a feature that
       enables them to keep running after exceptions that would
       otherwise kill their mainloop. This is a bother for IPython
       which excepts to catch all of the program exceptions with a try:
       except: statement.
 
       Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
       any app directly invokes sys.excepthook, it will look to the user like
       IPython crashed.  In order to work around this, we can disable the
       CrashHandler and replace it with this excepthook instead, which prints a
       regular traceback using our InteractiveTB.  In this fashion, apps which
       call sys.excepthook will generate a regular-looking exception from
       IPython, and the CrashHandler will only be triggered by real IPython
       crashes.
 
       This hook should be used sparingly, only in places which are not likely
       to be true IPython errors.
       """
       self.showtraceback((etype,value,tb),tb_offset=0)
 
     def _get_exc_info(self, exc_tuple=None):
         """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
         
         Ensures sys.last_type,value,traceback hold the exc_info we found,
         from whichever source.
         
         raises ValueError if none of these contain any information
         """
         if exc_tuple is None:
             etype, value, tb = sys.exc_info()
         else:
             etype, value, tb = exc_tuple
 
         if etype is None:
             if hasattr(sys, 'last_type'):
                 etype, value, tb = sys.last_type, sys.last_value, \
                                    sys.last_traceback
         
         if etype is None:
             raise ValueError("No exception to find")
         
         # Now store the exception info in sys.last_type etc.
         # WARNING: these variables are somewhat deprecated and not
         # necessarily safe to use in a threaded environment, but tools
         # like pdb depend on their existence, so let's set them.  If we
         # find problems in the field, we'll need to revisit their use.
         sys.last_type = etype
         sys.last_value = value
         sys.last_traceback = tb
         
         return etype, value, tb
     
     def show_usage_error(self, exc):
         """Show a short message for UsageErrors
         
         These are special exceptions that shouldn't show a traceback.
         """
         self.write_err("UsageError: %s" % exc)
     
     def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
                       exception_only=False):
         """Display the exception that just occurred.
 
         If nothing is known about the exception, this is the method which
         should be used throughout the code for presenting user tracebacks,
         rather than directly invoking the InteractiveTB object.
 
         A specific showsyntaxerror() also exists, but this method can take
         care of calling it if needed, so unless you are explicitly catching a
         SyntaxError exception, don't try to analyze the stack manually and
         simply call this method."""
 
         try:
             try:
                 etype, value, tb = self._get_exc_info(exc_tuple)
             except ValueError:
                 self.write_err('No traceback available to show.\n')
                 return
             
             if issubclass(etype, SyntaxError):
                 # Though this won't be called by syntax errors in the input
                 # line, there may be SyntaxError cases with imported code.
                 self.showsyntaxerror(filename)
             elif etype is UsageError:
                 self.show_usage_error(value)
             else:
                 if exception_only:
                     stb = ['An exception has occurred, use %tb to see '
                            'the full traceback.\n']
                     stb.extend(self.InteractiveTB.get_exception_only(etype,
                                                                      value))
                 else:
                     try:
                         # Exception classes can customise their traceback - we
                         # use this in IPython.parallel for exceptions occurring
                         # in the engines. This should return a list of strings.
                         stb = value._render_traceback_()
                     except Exception:
                         stb = self.InteractiveTB.structured_traceback(etype,
                                             value, tb, tb_offset=tb_offset)
 
                     self._showtraceback(etype, value, stb)
                     if self.call_pdb:
                         # drop into debugger
                         self.debugger(force=True)
                     return
 
                 # Actually show the traceback
                 self._showtraceback(etype, value, stb)
 
         except KeyboardInterrupt:
             self.write_err("\nKeyboardInterrupt\n")
 
     def _showtraceback(self, etype, evalue, stb):
         """Actually show a traceback.
 
         Subclasses may override this method to put the traceback on a different
         place, like a side channel.
         """
         print(self.InteractiveTB.stb2text(stb), file=io.stdout)
 
     def showsyntaxerror(self, filename=None):
         """Display the syntax error that just occurred.
 
         This doesn't display a stack trace because there isn't one.
 
         If a filename is given, it is stuffed in the exception instead
         of what was there before (because Python's parser always uses
         "<string>" when reading from a string).
         """
         etype, value, last_traceback = self._get_exc_info()
 
         if filename and issubclass(etype, SyntaxError):
             try:
                 value.filename = filename
             except:
                 # Not the format we expect; leave it alone
                 pass
         
         stb = self.SyntaxTB.structured_traceback(etype, value, [])
         self._showtraceback(etype, value, stb)
 
     # This is overridden in TerminalInteractiveShell to show a message about
     # the %paste magic.
     def showindentationerror(self):
         """Called by run_cell when there's an IndentationError in code entered
         at the prompt.
 
         This is overridden in TerminalInteractiveShell to show a message about
         the %paste magic."""
         self.showsyntaxerror()
 
     #-------------------------------------------------------------------------
     # Things related to readline
     #-------------------------------------------------------------------------
 
     def init_readline(self):
         """Command history completion/saving/reloading."""
 
         if self.readline_use:
             import IPython.utils.rlineimpl as readline
 
         self.rl_next_input = None
         self.rl_do_indent = False
 
         if not self.readline_use or not readline.have_readline:
             self.has_readline = False
             self.readline = None
             # Set a number of methods that depend on readline to be no-op
             self.readline_no_record = no_op_context
             self.set_readline_completer = no_op
             self.set_custom_completer = no_op
             if self.readline_use:
                 warn('Readline services not available or not loaded.')
         else:
             self.has_readline = True
             self.readline = readline
             sys.modules['readline'] = readline
 
             # Platform-specific configuration
             if os.name == 'nt':
                 # FIXME - check with Frederick to see if we can harmonize
                 # naming conventions with pyreadline to avoid this
                 # platform-dependent check
                 self.readline_startup_hook = readline.set_pre_input_hook
             else:
                 self.readline_startup_hook = readline.set_startup_hook
 
             # Load user's initrc file (readline config)
             # Or if libedit is used, load editrc.
             inputrc_name = os.environ.get('INPUTRC')
             if inputrc_name is None:
                 inputrc_name = '.inputrc'
                 if readline.uses_libedit:
                     inputrc_name = '.editrc'
                 inputrc_name = os.path.join(self.home_dir, inputrc_name)
             if os.path.isfile(inputrc_name):
                 try:
                     readline.read_init_file(inputrc_name)
                 except:
                     warn('Problems reading readline initialization file <%s>'
                          % inputrc_name)
 
             # Configure readline according to user's prefs
             # This is only done if GNU readline is being used.  If libedit
             # is being used (as on Leopard) the readline config is
             # not run as the syntax for libedit is different.
             if not readline.uses_libedit:
                 for rlcommand in self.readline_parse_and_bind:
                     #print "loading rl:",rlcommand  # dbg
                     readline.parse_and_bind(rlcommand)
 
             # Remove some chars from the delimiters list.  If we encounter
             # unicode chars, discard them.
             delims = readline.get_completer_delims()
             if not py3compat.PY3:
                 delims = delims.encode("ascii", "ignore")
             for d in self.readline_remove_delims:
                 delims = delims.replace(d, "")
             delims = delims.replace(ESC_MAGIC, '')
             readline.set_completer_delims(delims)
             # Store these so we can restore them if something like rpy2 modifies
             # them.
             self.readline_delims = delims
             # otherwise we end up with a monster history after a while:
             readline.set_history_length(self.history_length)
 
             self.refill_readline_hist()
             self.readline_no_record = ReadlineNoRecord(self)
 
         # Configure auto-indent for all platforms
         self.set_autoindent(self.autoindent)
 
     def refill_readline_hist(self):
         # Load the last 1000 lines from history
         self.readline.clear_history()
         stdin_encoding = sys.stdin.encoding or "utf-8"
         last_cell = u""
         for _, _, cell in self.history_manager.get_tail(1000,
                                                         include_latest=True):
             # Ignore blank lines and consecutive duplicates
             cell = cell.rstrip()
             if cell and (cell != last_cell):
                 try:
                     if self.multiline_history:
                           self.readline.add_history(py3compat.unicode_to_str(cell,
                                                                     stdin_encoding))
                     else:
                         for line in cell.splitlines():
                             self.readline.add_history(py3compat.unicode_to_str(line,
                                                                     stdin_encoding))
                     last_cell = cell
                 
                 except TypeError:
                     # The history DB can get corrupted so it returns strings
                     # containing null bytes, which readline objects to.
                     continue
 
     @skip_doctest
     def set_next_input(self, s):
         """ Sets the 'default' input string for the next command line.
 
         Requires readline.
 
         Example::
 
             In [1]: _ip.set_next_input("Hello Word")
             In [2]: Hello Word_  # cursor is here
         """
         self.rl_next_input = py3compat.cast_bytes_py2(s)
 
     # Maybe move this to the terminal subclass?
     def pre_readline(self):
         """readline hook to be used at the start of each line.
 
         Currently it handles auto-indent only."""
 
         if self.rl_do_indent:
             self.readline.insert_text(self._indent_current_str())
         if self.rl_next_input is not None:
             self.readline.insert_text(self.rl_next_input)
             self.rl_next_input = None
 
     def _indent_current_str(self):
         """return the current level of indentation as a string"""
         return self.input_splitter.indent_spaces * ' '
 
     #-------------------------------------------------------------------------
     # Things related to text completion
     #-------------------------------------------------------------------------
 
     def init_completer(self):
         """Initialize the completion machinery.
 
         This creates completion machinery that can be used by client code,
         either interactively in-process (typically triggered by the readline
         library), programatically (such as in test suites) or out-of-prcess
         (typically over the network by remote frontends).
         """
         from IPython.core.completer import IPCompleter
         from IPython.core.completerlib import (module_completer,
                 magic_run_completer, cd_completer, reset_completer)
 
         self.Completer = IPCompleter(shell=self,
                                      namespace=self.user_ns,
                                      global_namespace=self.user_global_ns,
                                      use_readline=self.has_readline,
                                      parent=self,
                                      )
         self.configurables.append(self.Completer)
 
         # Add custom completers to the basic ones built into IPCompleter
         sdisp = self.strdispatchers.get('complete_command', StrDispatch())
         self.strdispatchers['complete_command'] = sdisp
         self.Completer.custom_completers = sdisp
 
         self.set_hook('complete_command', module_completer, str_key = 'import')
         self.set_hook('complete_command', module_completer, str_key = 'from')
         self.set_hook('complete_command', magic_run_completer, str_key = '%run')
         self.set_hook('complete_command', cd_completer, str_key = '%cd')
         self.set_hook('complete_command', reset_completer, str_key = '%reset')
 
         # Only configure readline if we truly are using readline.  IPython can
         # do tab-completion over the network, in GUIs, etc, where readline
         # itself may be absent
         if self.has_readline:
             self.set_readline_completer()
 
     def complete(self, text, line=None, cursor_pos=None):
         """Return the completed text and a list of completions.
 
         Parameters
         ----------
 
            text : string
              A string of text to be completed on.  It can be given as empty and
              instead a line/position pair are given.  In this case, the
              completer itself will split the line like readline does.
 
            line : string, optional
              The complete line that text is part of.
 
            cursor_pos : int, optional
              The position of the cursor on the input line.
 
         Returns
         -------
           text : string
             The actual text that was completed.
 
           matches : list
             A sorted list with all possible completions.
 
         The optional arguments allow the completion to take more context into
         account, and are part of the low-level completion API.
 
         This is a wrapper around the completion mechanism, similar to what
         readline does at the command line when the TAB key is hit.  By
         exposing it as a method, it can be used by other non-readline
         environments (such as GUIs) for text completion.
 
         Simple usage example:
 
         In [1]: x = 'hello'
 
         In [2]: _ip.complete('x.l')
         Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
         """
 
         # Inject names into __builtin__ so we can complete on the added names.
         with self.builtin_trap:
             return self.Completer.complete(text, line, cursor_pos)
 
     def set_custom_completer(self, completer, pos=0):
         """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted."""
 
         newcomp = types.MethodType(completer,self.Completer)
         self.Completer.matchers.insert(pos,newcomp)
 
     def set_readline_completer(self):
         """Reset readline's completer to be our own."""
         self.readline.set_completer(self.Completer.rlcomplete)
 
     def set_completer_frame(self, frame=None):
         """Set the frame of the completer."""
         if frame:
             self.Completer.namespace = frame.f_locals
             self.Completer.global_namespace = frame.f_globals
         else:
             self.Completer.namespace = self.user_ns
             self.Completer.global_namespace = self.user_global_ns
 
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         from IPython.core import magics as m
         self.magics_manager = magic.MagicsManager(shell=self,
                                    parent=self,
                                    user_magics=m.UserMagics(self))
         self.configurables.append(self.magics_manager)
 
         # Expose as public API from the magics manager
         self.register_magics = self.magics_manager.register
         self.define_magic = self.magics_manager.define_magic
 
         self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
             m.ConfigMagics, m.DeprecatedMagics, m.DisplayMagics, m.ExecutionMagics,
             m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
             m.NamespaceMagics, m.OSMagics, m.PylabMagics, m.ScriptMagics,
         )
 
         # Register Magic Aliases
         mman = self.magics_manager
         # FIXME: magic aliases should be defined by the Magics classes
         # or in MagicsManager, not here
         mman.register_alias('ed', 'edit')
         mman.register_alias('hist', 'history')
         mman.register_alias('rep', 'recall')
         mman.register_alias('SVG', 'svg', 'cell')
         mman.register_alias('HTML', 'html', 'cell')
         mman.register_alias('file', 'writefile', 'cell')
 
         # FIXME: Move the color initialization to the DisplayHook, which
         # should be split into a prompt manager and displayhook. We probably
         # even need a centralize colors management object.
         self.magic('colors %s' % self.colors)
     
     # Defined here so that it's included in the documentation
     @functools.wraps(magic.MagicsManager.register_function)
     def register_magic_function(self, func, magic_kind='line', magic_name=None):
         self.magics_manager.register_function(func, 
                                   magic_kind=magic_kind, magic_name=magic_name)
 
     def run_line_magic(self, magic_name, line):
         """Execute the given line magic.
 
         Parameters
         ----------
         magic_name : str
           Name of the desired magic function, without '%' prefix.
 
         line : str
           The rest of the input line as a single string.
         """
         fn = self.find_line_magic(magic_name)
         if fn is None:
             cm = self.find_cell_magic(magic_name)
             etpl = "Line magic function `%%%s` not found%s."
             extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
                                     'did you mean that instead?)' % magic_name )
             error(etpl % (magic_name, extra))
         else:
             # Note: this is the distance in the stack to the user's frame.
             # This will need to be updated if the internal calling logic gets
             # refactored, or else we'll be expanding the wrong variables.
             stack_depth = 2
             magic_arg_s = self.var_expand(line, stack_depth)
             # Put magic args in a list so we can call with f(*a) syntax
             args = [magic_arg_s]
             kwargs = {}
             # Grab local namespace if we need it:
             if getattr(fn, "needs_local_scope", False):
                 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
             with self.builtin_trap:
                 result = fn(*args,**kwargs)
             return result
 
     def run_cell_magic(self, magic_name, line, cell):
         """Execute the given cell magic.
         
         Parameters
         ----------
         magic_name : str
           Name of the desired magic function, without '%' prefix.
 
         line : str
           The rest of the first input line as a single string.
 
         cell : str
           The body of the cell as a (possibly multiline) string.
         """
         fn = self.find_cell_magic(magic_name)
         if fn is None:
             lm = self.find_line_magic(magic_name)
             etpl = "Cell magic `%%{0}` not found{1}."
             extra = '' if lm is None else (' (But line magic `%{0}` exists, '
                             'did you mean that instead?)'.format(magic_name))
             error(etpl.format(magic_name, extra))
         elif cell == '':
             message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
             if self.find_line_magic(magic_name) is not None:
                 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
             raise UsageError(message)
         else:
             # Note: this is the distance in the stack to the user's frame.
             # This will need to be updated if the internal calling logic gets
             # refactored, or else we'll be expanding the wrong variables.
             stack_depth = 2
             magic_arg_s = self.var_expand(line, stack_depth)
             with self.builtin_trap:
                 result = fn(magic_arg_s, cell)
             return result
 
     def find_line_magic(self, magic_name):
         """Find and return a line magic by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics['line'].get(magic_name)
 
     def find_cell_magic(self, magic_name):
         """Find and return a cell magic by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics['cell'].get(magic_name)
 
     def find_magic(self, magic_name, magic_kind='line'):
         """Find and return a magic of the given type by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics[magic_kind].get(magic_name)
 
     def magic(self, arg_s):
         """DEPRECATED. Use run_line_magic() instead.
 
         Call a magic function by name.
 
         Input: a string containing the name of the magic function to call and
         any additional arguments to be passed to the magic.
 
         magic('name -opt foo bar') is equivalent to typing at the ipython
         prompt:
 
         In[1]: %name -opt foo bar
 
         To call a magic without arguments, simply use magic('name').
 
         This provides a proper Python function to call IPython's magics in any
         valid Python code you can type at the interpreter, including loops and
         compound statements.
         """
         # TODO: should we issue a loud deprecation warning here?
         magic_name, _, magic_arg_s = arg_s.partition(' ')
         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
         return self.run_line_magic(magic_name, magic_arg_s)
 
     #-------------------------------------------------------------------------
     # Things related to macros
     #-------------------------------------------------------------------------
 
     def define_macro(self, name, themacro):
         """Define a new macro
 
         Parameters
         ----------
         name : str
             The name of the macro.
         themacro : str or Macro
             The action to do upon invoking the macro.  If a string, a new
             Macro object is created by passing the string to it.
         """
 
         from IPython.core import macro
 
         if isinstance(themacro, string_types):
             themacro = macro.Macro(themacro)
         if not isinstance(themacro, macro.Macro):
             raise ValueError('A macro must be a string or a Macro instance.')
         self.user_ns[name] = themacro
 
     #-------------------------------------------------------------------------
     # Things related to the running of system commands
     #-------------------------------------------------------------------------
 
     def system_piped(self, cmd):
         """Call the given cmd in a subprocess, piping stdout/err
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as background processes are
           not supported.  Should not be a command that expects input
           other than simple text.
         """
         if cmd.rstrip().endswith('&'):
             # this is *far* from a rigorous test
             # We do not support backgrounding processes because we either use
             # pexpect or pipes to read from.  Users can always just call
             # os.system() or use ip.system=ip.system_raw
             # if they really want a background process.
             raise OSError("Background processes not supported.")
 
         # we explicitly do NOT return the subprocess status code, because
         # a non-None value would trigger :func:`sys.displayhook` calls.
         # Instead, we store the exit_code in user_ns.
         self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
 
     def system_raw(self, cmd):
         """Call the given cmd in a subprocess using os.system on Windows or
         subprocess.call using the system shell on other platforms.
 
         Parameters
         ----------
         cmd : str
           Command to execute.
         """
         cmd = self.var_expand(cmd, depth=1)
         # protect os.system from UNC paths on Windows, which it can't handle:
         if sys.platform == 'win32':
             from IPython.utils._process_win32 import AvoidUNCPath
             with AvoidUNCPath() as path:
                 if path is not None:
                     cmd = '"pushd %s &&"%s' % (path, cmd)
                 cmd = py3compat.unicode_to_str(cmd)
                 ec = os.system(cmd)
         else:
             cmd = py3compat.unicode_to_str(cmd)
             # Call the cmd using the OS shell, instead of the default /bin/sh, if set.
             ec = subprocess.call(cmd, shell=True, executable=os.environ.get('SHELL', None))
             # exit code is positive for program failure, or negative for
             # terminating signal number.
         
         # We explicitly do NOT return the subprocess status code, because
         # a non-None value would trigger :func:`sys.displayhook` calls.
         # Instead, we store the exit_code in user_ns.
         self.user_ns['_exit_code'] = ec
 
     # use piped system by default, because it is better behaved
     system = system_piped
 
     def getoutput(self, cmd, split=True, depth=0):
         """Get output (possibly including stderr) from a subprocess.
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as background processes are
           not supported.
         split : bool, optional
           If True, split the output into an IPython SList.  Otherwise, an
           IPython LSString is returned.  These are objects similar to normal
           lists and strings, with a few convenience attributes for easier
           manipulation of line-based output.  You can use '?' on them for
           details.
         depth : int, optional
           How many frames above the caller are the local variables which should
           be expanded in the command string? The default (0) assumes that the
           expansion variables are in the stack frame calling this function.
         """
         if cmd.rstrip().endswith('&'):
             # this is *far* from a rigorous test
             raise OSError("Background processes not supported.")
         out = getoutput(self.var_expand(cmd, depth=depth+1))
         if split:
             out = SList(out.splitlines())
         else:
             out = LSString(out)
         return out
 
     #-------------------------------------------------------------------------
     # Things related to aliases
     #-------------------------------------------------------------------------
 
     def init_alias(self):
         self.alias_manager = AliasManager(shell=self, parent=self)
         self.configurables.append(self.alias_manager)
 
     #-------------------------------------------------------------------------
     # Things related to extensions
     #-------------------------------------------------------------------------
 
     def init_extension_manager(self):
         self.extension_manager = ExtensionManager(shell=self, parent=self)
         self.configurables.append(self.extension_manager)
 
     #-------------------------------------------------------------------------
     # Things related to payloads
     #-------------------------------------------------------------------------
 
     def init_payload(self):
         self.payload_manager = PayloadManager(parent=self)
         self.configurables.append(self.payload_manager)
     
     #-------------------------------------------------------------------------
     # Things related to widgets
     #-------------------------------------------------------------------------
     
     def init_comms(self):
         # not implemented in the base class
         pass
     
     #-------------------------------------------------------------------------
     # Things related to the prefilter
     #-------------------------------------------------------------------------
 
     def init_prefilter(self):
         self.prefilter_manager = PrefilterManager(shell=self, parent=self)
         self.configurables.append(self.prefilter_manager)
         # Ultimately this will be refactored in the new interpreter code, but
         # for now, we should expose the main prefilter method (there's legacy
         # code out there that may rely on this).
         self.prefilter = self.prefilter_manager.prefilter_lines
 
     def auto_rewrite_input(self, cmd):
         """Print to the screen the rewritten form of the user's command.
 
         This shows visual feedback by rewriting input lines that cause
         automatic calling to kick in, like::
 
           /f x
 
         into::
 
           ------> f(x)
 
         after the user's input prompt.  This helps the user understand that the
         input line was transformed automatically by IPython.
         """
         if not self.show_rewritten_input:
             return
         
         rw = self.prompt_manager.render('rewrite') + cmd
 
         try:
             # plain ascii works better w/ pyreadline, on some machines, so
             # we use it and only print uncolored rewrite if we have unicode
             rw = str(rw)
             print(rw, file=io.stdout)
         except UnicodeEncodeError:
             print("------> " + cmd)
 
     #-------------------------------------------------------------------------
     # Things related to extracting values/expressions from kernel and user_ns
     #-------------------------------------------------------------------------
 
     def _user_obj_error(self):
         """return simple exception dict
         
         for use in user_variables / expressions
         """
         
         etype, evalue, tb = self._get_exc_info()
         stb = self.InteractiveTB.get_exception_only(etype, evalue)
         
         exc_info = {
             u'status' : 'error',
             u'traceback' : stb,
             u'ename' : unicode_type(etype.__name__),
             u'evalue' : py3compat.safe_unicode(evalue),
         }
 
         return exc_info
     
     def _format_user_obj(self, obj):
         """format a user object to display dict
         
         for use in user_expressions / variables
         """
         
         data, md = self.display_formatter.format(obj)
         value = {
             'status' : 'ok',
             'data' : data,
             'metadata' : md,
         }
         return value
     
     def user_variables(self, names):
         """Get a list of variable names from the user's namespace.
 
         Parameters
         ----------
         names : list of strings
           A list of names of variables to be read from the user namespace.
 
         Returns
         -------
         A dict, keyed by the input names and with the rich mime-type repr(s) of each value.
         Each element will be a sub-dict of the same form as a display_data message.
         """
         out = {}
         user_ns = self.user_ns
         
         for varname in names:
             try:
                 value = self._format_user_obj(user_ns[varname])
             except:
                 value = self._user_obj_error()
             out[varname] = value
         return out
 
     def user_expressions(self, expressions):
         """Evaluate a dict of expressions in the user's namespace.
 
         Parameters
         ----------
         expressions : dict
           A dict with string keys and string values.  The expression values
           should be valid Python expressions, each of which will be evaluated
           in the user namespace.
 
         Returns
         -------
         A dict, keyed like the input expressions dict, with the rich mime-typed
         display_data of each value.
         """
         out = {}
         user_ns = self.user_ns
         global_ns = self.user_global_ns
         
         for key, expr in iteritems(expressions):
             try:
                 value = self._format_user_obj(eval(expr, global_ns, user_ns))
             except:
                 value = self._user_obj_error()
             out[key] = value
         return out
 
     #-------------------------------------------------------------------------
     # Things related to the running of code
     #-------------------------------------------------------------------------
 
     def ex(self, cmd):
         """Execute a normal python statement in user namespace."""
         with self.builtin_trap:
             exec(cmd, self.user_global_ns, self.user_ns)
 
     def ev(self, expr):
         """Evaluate python expression expr in user namespace.
 
         Returns the result of evaluation
         """
         with self.builtin_trap:
             return eval(expr, self.user_global_ns, self.user_ns)
 
     def safe_execfile(self, fname, *where, **kw):
         """A safe version of the builtin execfile().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.  This only works on pure
         Python files with the .py extension.
 
         Parameters
         ----------
         fname : string
             The name of the file to be executed.
         where : tuple
             One or two namespaces, passed to execfile() as (globals,locals).
             If only one is given, it is passed as both.
         exit_ignore : bool (False)
             If True, then silence SystemExit for non-zero status (it is always
             silenced for zero status, as it is so common).
         raise_exceptions : bool (False)
             If True raise exceptions everywhere. Meant for testing.
 
         """
         kw.setdefault('exit_ignore', False)
         kw.setdefault('raise_exceptions', False)
 
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we can open the file
         try:
             with open(fname) as thefile:
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
 
         with prepended_to_syspath(dname):
             try:
                 py3compat.execfile(fname,*where)
             except SystemExit as status:
                 # If the call was made with 0 or None exit status (sys.exit(0)
                 # or sys.exit() ), don't bother showing a traceback, as both of
                 # these are considered normal by the OS:
                 # > python -c'import sys;sys.exit(0)'; echo $?
                 # 0
                 # > python -c'import sys;sys.exit()'; echo $?
                 # 0
                 # For other exit status, we show the exception unless
                 # explicitly silenced, but only in short form.
                 if kw['raise_exceptions']:
                     raise
                 if status.code and not kw['exit_ignore']:
                     self.showtraceback(exception_only=True)
             except:
                 if kw['raise_exceptions']:
                     raise
                 self.showtraceback()
 
     def safe_execfile_ipy(self, fname):
         """Like safe_execfile, but for .ipy files with IPython syntax.
 
         Parameters
         ----------
         fname : str
             The name of the file to execute.  The filename must have a
             .ipy extension.
         """
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we can open the file
         try:
             with open(fname) as thefile:
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
 
         with prepended_to_syspath(dname):
             try:
                 with open(fname) as thefile:
                     # self.run_cell currently captures all exceptions
                     # raised in user code.  It would be nice if there were
                     # versions of runlines, execfile that did raise, so
                     # we could catch the errors.
                     self.run_cell(thefile.read(), store_history=False, shell_futures=False)
             except:
                 self.showtraceback()
                 warn('Unknown failure executing file: <%s>' % fname)
 
     def safe_run_module(self, mod_name, where):
         """A safe version of runpy.run_module().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.
 
         `SystemExit` exceptions with status code 0 or None are ignored.
 
         Parameters
         ----------
         mod_name : string
             The name of the module to be executed.
         where : dict
             The globals namespace.
         """
         try:
             try:
                 where.update(
                     runpy.run_module(str(mod_name), run_name="__main__",
                                      alter_sys=True)
                     )
             except SystemExit as status:
                 if status.code:
                     raise
         except:
             self.showtraceback()
             warn('Unknown failure executing module: <%s>' % mod_name)
 
     def _run_cached_cell_magic(self, magic_name, line):
         """Special method to call a cell magic with the data stored in self.
         """
         cell = self._current_cell_magic_body
         self._current_cell_magic_body = None
         return self.run_cell_magic(magic_name, line, cell)
 
     def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True):
         """Run a complete IPython cell.
 
         Parameters
         ----------
         raw_cell : str
           The code (including IPython code such as %magic functions) to run.
         store_history : bool
           If True, the raw and translated cell will be stored in IPython's
           history. For user code calling back into IPython's machinery, this
           should be set to False.
         silent : bool
           If True, avoid side-effects, such as implicit displayhooks and
           and logging.  silent=True forces store_history=False.
         shell_futures : bool
           If True, the code will share future statements with the interactive
           shell. It will both be affected by previous __future__ imports, and
           any __future__ imports in the code will affect the shell. If False,
           __future__ imports are not shared in either direction.
         """
         if (not raw_cell) or raw_cell.isspace():
             return
         
         if silent:
             store_history = False
 
         self.input_transformer_manager.push(raw_cell)
         cell = self.input_transformer_manager.source_reset()
 
         # Our own compiler remembers the __future__ environment. If we want to
         # run code with a separate __future__ environment, use the default
         # compiler
         compiler = self.compile if shell_futures else CachingCompiler()
 
         with self.builtin_trap:
             prefilter_failed = False
             if len(cell.splitlines()) == 1:
                 try:
                     # use prefilter_lines to handle trailing newlines
                     # restore trailing newline for ast.parse
                     cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
                 except AliasError as e:
                     error(e)
                     prefilter_failed = True
                 except Exception:
                     # don't allow prefilter errors to crash IPython
                     self.showtraceback()
                     prefilter_failed = True
 
             # Store raw and processed history
             if store_history:
                 self.history_manager.store_inputs(self.execution_count,
                                                   cell, raw_cell)
             if not silent:
                 self.logger.log(cell, raw_cell)
 
             if not prefilter_failed:
                 # don't run if prefilter failed
                 cell_name = self.compile.cache(cell, self.execution_count)
 
                 with self.display_trap:
                     try:
                         code_ast = compiler.ast_parse(cell, filename=cell_name)
                     except IndentationError:
                         self.showindentationerror()
                         if store_history:
                             self.execution_count += 1
                         return None
                     except (OverflowError, SyntaxError, ValueError, TypeError,
                             MemoryError):
                         self.showsyntaxerror()
                         if store_history:
                             self.execution_count += 1
                         return None
                     
                     code_ast = self.transform_ast(code_ast)
                     
                     interactivity = "none" if silent else self.ast_node_interactivity
                     self.run_ast_nodes(code_ast.body, cell_name,
                                        interactivity=interactivity, compiler=compiler)
                     
                     # Execute any registered post-execution functions.
                     # unless we are silent
                     post_exec = [] if silent else iteritems(self._post_execute)
                     
                     for func, status in post_exec:
                         if self.disable_failing_post_execute and not status:
                             continue
                         try:
                             func()
                         except KeyboardInterrupt:
                             print("\nKeyboardInterrupt", file=io.stderr)
                         except Exception:
                             # register as failing:
                             self._post_execute[func] = False
                             self.showtraceback()
                             print('\n'.join([
                                 "post-execution function %r produced an error." % func,
                                 "If this problem persists, you can disable failing post-exec functions with:",
                                 "",
                                 "    get_ipython().disable_failing_post_execute = True"
                             ]), file=io.stderr)
 
         if store_history:
             # Write output to the database. Does nothing unless
             # history output logging is enabled.
             self.history_manager.store_output(self.execution_count)
             # Each cell is a *single* input, regardless of how many lines it has
             self.execution_count += 1
     
     def transform_ast(self, node):
         """Apply the AST transformations from self.ast_transformers
         
         Parameters
         ----------
         node : ast.Node
           The root node to be transformed. Typically called with the ast.Module
           produced by parsing user input.
         
         Returns
         -------
         An ast.Node corresponding to the node it was called with. Note that it
         may also modify the passed object, so don't rely on references to the
         original AST.
         """
         for transformer in self.ast_transformers:
             try:
                 node = transformer.visit(node)
             except Exception:
                 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
                 self.ast_transformers.remove(transformer)
         
         if self.ast_transformers:
             ast.fix_missing_locations(node)
         return node
                 
 
     def run_ast_nodes(self, nodelist, cell_name, interactivity='last_expr',
                         compiler=compile):
         """Run a sequence of AST nodes. The execution mode depends on the
         interactivity parameter.
 
         Parameters
         ----------
         nodelist : list
           A sequence of AST nodes to run.
         cell_name : str
           Will be passed to the compiler as the filename of the cell. Typically
           the value returned by ip.compile.cache(cell).
         interactivity : str
           'all', 'last', 'last_expr' or 'none', specifying which nodes should be
           run interactively (displaying output from expressions). 'last_expr'
           will run the last node interactively only if it is an expression (i.e.
           expressions in loops or other blocks are not displayed. Other values
           for this parameter will raise a ValueError.
         compiler : callable
           A function with the same interface as the built-in compile(), to turn
           the AST nodes into code objects. Default is the built-in compile().
         """
         if not nodelist:
             return
 
         if interactivity == 'last_expr':
             if isinstance(nodelist[-1], ast.Expr):
                 interactivity = "last"
             else:
                 interactivity = "none"
 
         if interactivity == 'none':
             to_run_exec, to_run_interactive = nodelist, []
         elif interactivity == 'last':
             to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
         elif interactivity == 'all':
             to_run_exec, to_run_interactive = [], nodelist
         else:
             raise ValueError("Interactivity was %r" % interactivity)
 
         exec_count = self.execution_count
 
         try:
             for i, node in enumerate(to_run_exec):
                 mod = ast.Module([node])
                 code = compiler(mod, cell_name, "exec")
                 if self.run_code(code):
                     return True
 
             for i, node in enumerate(to_run_interactive):
                 mod = ast.Interactive([node])
                 code = compiler(mod, cell_name, "single")
                 if self.run_code(code):
                     return True
 
             # Flush softspace
             if softspace(sys.stdout, 0):
                 print()
 
         except:
             # It's possible to have exceptions raised here, typically by
             # compilation of odd code (such as a naked 'return' outside a
             # function) that did parse but isn't valid. Typically the exception
             # is a SyntaxError, but it's safest just to catch anything and show
             # the user a traceback.
 
             # We do only one try/except outside the loop to minimize the impact
             # on runtime, and also because if any node in the node list is
             # broken, we should stop execution completely.
             self.showtraceback()
 
         return False
 
     def run_code(self, code_obj):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
         traceback.
 
         Parameters
         ----------
         code_obj : code object
           A compiled code object, to be executed
 
         Returns
         -------
         False : successful execution.
         True : an error occurred.
         """
 
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
 
         # we save the original sys.excepthook in the instance, in case config
         # code (such as magics) needs access to it.
         self.sys_excepthook = old_excepthook
         outflag = 1  # happens in more places, so it's easier as default
         try:
             try:
                 self.hooks.pre_run_code_hook()
                 #rprint('Running code', repr(code_obj)) # dbg
                 exec(code_obj, self.user_global_ns, self.user_ns)
             finally:
                 # Reset our crash handler in place
                 sys.excepthook = old_excepthook
         except SystemExit:
             self.showtraceback(exception_only=True)
             warn("To exit: use 'exit', 'quit', or Ctrl-D.", level=1)
         except self.custom_exceptions:
             etype,value,tb = sys.exc_info()
             self.CustomTB(etype,value,tb)
         except:
             self.showtraceback()
         else:
             outflag = 0
         return outflag
 
     # For backwards compatibility
     runcode = run_code
 
     #-------------------------------------------------------------------------
     # Things related to GUI support and pylab
     #-------------------------------------------------------------------------
 
     def enable_gui(self, gui=None):
         raise NotImplementedError('Implement enable_gui in a subclass')
     
     def enable_matplotlib(self, gui=None):
         """Enable interactive matplotlib and inline figure support.
         
         This takes the following steps:
         
         1. select the appropriate eventloop and matplotlib backend
         2. set up matplotlib for interactive use with that backend
         3. configure formatters for inline figure display
         4. enable the selected gui eventloop
         
         Parameters
         ----------
         gui : optional, string
           If given, dictates the choice of matplotlib GUI backend to use
           (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
           'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
           matplotlib (as dictated by the matplotlib build-time options plus the
           user's matplotlibrc configuration file).  Note that not all backends
           make sense in all contexts, for example a terminal ipython can't
           display figures inline.
         """
         from IPython.core import pylabtools as pt
         gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
     
         if gui != 'inline':
             # If we have our first gui selection, store it
             if self.pylab_gui_select is None:
                 self.pylab_gui_select = gui
             # Otherwise if they are different
             elif gui != self.pylab_gui_select:
                 print ('Warning: Cannot change to a different GUI toolkit: %s.'
                         ' Using %s instead.' % (gui, self.pylab_gui_select))
                 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
         
         pt.activate_matplotlib(backend)
         pt.configure_inline_support(self, backend)
         
         # Now we must activate the gui pylab wants to use, and fix %run to take
         # plot updates into account
         self.enable_gui(gui)
         self.magics_manager.registry['ExecutionMagics'].default_runner = \
             pt.mpl_runner(self.safe_execfile)
         
         return gui, backend
 
     def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
         """Activate pylab support at runtime.
 
         This turns on support for matplotlib, preloads into the interactive
         namespace all of numpy and pylab, and configures IPython to correctly
         interact with the GUI event loop.  The GUI backend to be used can be
         optionally selected with the optional ``gui`` argument.
         
         This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
 
         Parameters
         ----------
         gui : optional, string
           If given, dictates the choice of matplotlib GUI backend to use
           (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
           'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
           matplotlib (as dictated by the matplotlib build-time options plus the
           user's matplotlibrc configuration file).  Note that not all backends
           make sense in all contexts, for example a terminal ipython can't
           display figures inline.
         import_all : optional, bool, default: True
           Whether to do `from numpy import *` and `from pylab import *`
           in addition to module imports.
         welcome_message : deprecated
           This argument is ignored, no welcome message will be displayed.
         """
         from IPython.core.pylabtools import import_pylab
         
         gui, backend = self.enable_matplotlib(gui)
         
         # We want to prevent the loading of pylab to pollute the user's
         # namespace as shown by the %who* magics, so we execute the activation
         # code in an empty namespace, and we update *both* user_ns and
         # user_ns_hidden with this information.
         ns = {}
         import_pylab(ns, import_all)
         # warn about clobbered names
         ignored = set(["__builtins__"])
         both = set(ns).intersection(self.user_ns).difference(ignored)
         clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
         self.user_ns.update(ns)
         self.user_ns_hidden.update(ns)
         return gui, backend, clobbered
 
     #-------------------------------------------------------------------------
     # Utilities
     #-------------------------------------------------------------------------
 
     def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
         """Expand python variables in a string.
 
         The depth argument indicates how many frames above the caller should
         be walked to look for the local namespace where to expand variables.
 
         The global namespace for expansion is always the user's interactive
         namespace.
         """
         ns = self.user_ns.copy()
         ns.update(sys._getframe(depth+1).f_locals)
         try:
             # We have to use .vformat() here, because 'self' is a valid and common
             # name, and expanding **ns for .format() would make it collide with
             # the 'self' argument of the method.
             cmd = formatter.vformat(cmd, args=[], kwargs=ns)
         except Exception:
             # if formatter couldn't format, just let it go untransformed
             pass
         return cmd
 
     def mktempfile(self, data=None, prefix='ipython_edit_'):
         """Make a new tempfile and return its filename.
 
         This makes a call to tempfile.mktemp, but it registers the created
         filename internally so ipython cleans it up at exit time.
 
         Optional inputs:
 
           - data(None): if data is given, it gets written out to the temp file
             immediately, and the file is closed again."""
 
         filename = tempfile.mktemp('.py', prefix)
         self.tempfiles.append(filename)
 
         if data:
             tmp_file = open(filename,'w')
             tmp_file.write(data)
             tmp_file.close()
         return filename
 
     # TODO:  This should be removed when Term is refactored.
     def write(self,data):
         """Write a string to the default output"""
         io.stdout.write(data)
 
     # TODO:  This should be removed when Term is refactored.
     def write_err(self,data):
         """Write a string to the default error output"""
         io.stderr.write(data)
 
     def ask_yes_no(self, prompt, default=None):
         if self.quiet:
             return True
         return ask_yes_no(prompt,default)
 
     def show_usage(self):
         """Show a usage message"""
         page.page(IPython.core.usage.interactive_usage)
 
     def extract_input_lines(self, range_str, raw=False):
         """Return as a string a set of input history slices.
 
         Parameters
         ----------
         range_str : string
             The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
             since this function is for use by magic functions which get their
             arguments as strings. The number before the / is the session
             number: ~n goes n back from the current session.
 
         Optional Parameters:
           - raw(False): by default, the processed input is used.  If this is
             true, the raw input history is used instead.
 
         Note that slices can be called with two notations:
 
         N:M -> standard python form, means including items N...(M-1).
 
         N-M -> include items N..M (closed endpoint).
         """
         lines = self.history_manager.get_range_by_str(range_str, raw=raw)
         return "\n".join(x for _, _, x in lines)
 
     def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True):
         """Get a code string from history, file, url, or a string or macro.
 
         This is mainly used by magic functions.
 
         Parameters
         ----------
 
         target : str
 
           A string specifying code to retrieve. This will be tried respectively
           as: ranges of input history (see %history for syntax), url,
           correspnding .py file, filename, or an expression evaluating to a
           string or Macro in the user namespace.
 
         raw : bool
           If true (default), retrieve raw history. Has no effect on the other
           retrieval mechanisms.
 
         py_only : bool (default False)
           Only try to fetch python code, do not try alternative methods to decode file
           if unicode fails.
 
         Returns
         -------
         A string of code.
 
         ValueError is raised if nothing is found, and TypeError if it evaluates
         to an object of another type. In each case, .args[0] is a printable
         message.
         """
         code = self.extract_input_lines(target, raw=raw)  # Grab history
         if code:
             return code
         utarget = unquote_filename(target)
         try:
             if utarget.startswith(('http://', 'https://')):
                 return openpy.read_py_url(utarget, skip_encoding_cookie=skip_encoding_cookie)
         except UnicodeDecodeError:
             if not py_only :
                 from urllib import urlopen  # Deferred import
                 response = urlopen(target)
                 return response.read().decode('latin1')
             raise ValueError(("'%s' seem to be unreadable.") % utarget)
 
         potential_target = [target]
         try :
             potential_target.insert(0,get_py_filename(target))
         except IOError:
             pass
 
         for tgt in potential_target :
             if os.path.isfile(tgt):                        # Read file
                 try :
                     return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
                 except UnicodeDecodeError :
                     if not py_only :
                         with io_open(tgt,'r', encoding='latin1') as f :
                             return f.read()
                     raise ValueError(("'%s' seem to be unreadable.") % target)
             elif os.path.isdir(os.path.expanduser(tgt)):
                 raise ValueError("'%s' is a directory, not a regular file." % target)
 
         try:                                              # User namespace
             codeobj = eval(target, self.user_ns)
         except Exception:
             raise ValueError(("'%s' was not found in history, as a file, url, "
                                 "nor in the user namespace.") % target)
         if isinstance(codeobj, string_types):
             return codeobj
         elif isinstance(codeobj, Macro):
             return codeobj.value
 
         raise TypeError("%s is neither a string nor a macro." % target,
                         codeobj)
 
     #-------------------------------------------------------------------------
     # Things related to IPython exiting
     #-------------------------------------------------------------------------
     def atexit_operations(self):
         """This will be executed at the time of exit.
 
         Cleanup operations and saving of persistent data that is done
         unconditionally by IPython should be performed here.
 
         For things that may depend on startup flags or platform specifics (such
         as having readline or not), register a separate atexit function in the
         code that has the appropriate information, rather than trying to
         clutter
         """
         # Close the history session (this stores the end time and line count)
         # this must be *before* the tempfile cleanup, in case of temporary
         # history db
         self.history_manager.end_session()
 
         # Cleanup all tempfiles left around
         for tfile in self.tempfiles:
             try:
                 os.unlink(tfile)
             except OSError:
                 pass
 
         # Clear all user namespaces to release all references cleanly.
         self.reset(new_session=False)
 
         # Run user hooks
         self.hooks.shutdown_hook()
 
     def cleanup(self):
         self.restore_sys_module_state()
 
 
 class InteractiveShellABC(with_metaclass(abc.ABCMeta, object)):
     """An abstract base class for InteractiveShell."""
 
 InteractiveShellABC.register(InteractiveShell)

IPython/core/magics/osm.py

@@ -1,740 +1,741 b''
 """Implementation of magic functions for interaction with the OS.
 
 Note: this module is named 'osm' instead of 'os' to avoid a collision with the
 builtin.
 """
 from __future__ import print_function
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import io
 import os
 import re
 import sys
 from pprint import pformat
 
 # Our own packages
 from IPython.core import magic_arguments
 from IPython.core import oinspect
 from IPython.core import page
 from IPython.core.alias import AliasError, Alias
 from IPython.core.error import UsageError
 from IPython.core.magic import  (
     Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
 )
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.openpy import source_to_unicode
 from IPython.utils.path import unquote_filename
 from IPython.utils.process import abbrev_cwd
+from IPython.utils import py3compat
 from IPython.utils.py3compat import unicode_type
 from IPython.utils.terminal import set_term_title
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 @magics_class
 class OSMagics(Magics):
     """Magics to interact with the underlying OS (shell-type functionality).
     """
 
     @skip_doctest
     @line_magic
     def alias(self, parameter_s=''):
         """Define an alias for a system command.
 
         '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
 
         Then, typing 'alias_name params' will execute the system command 'cmd
         params' (from your underlying operating system).
 
         Aliases have lower precedence than magic functions and Python normal
         variables, so if 'foo' is both a Python variable and an alias, the
         alias can not be executed until 'del foo' removes the Python variable.
 
         You can use the %l specifier in an alias definition to represent the
         whole line when the alias is called.  For example::
 
           In [2]: alias bracket echo "Input in brackets: <%l>"
           In [3]: bracket hello world
           Input in brackets: <hello world>
 
         You can also define aliases with parameters using %s specifiers (one
         per parameter)::
 
           In [1]: alias parts echo first %s second %s
           In [2]: %parts A B
           first A second B
           In [3]: %parts A
           Incorrect number of arguments: 2 expected.
           parts is an alias to: 'echo first %s second %s'
 
         Note that %l and %s are mutually exclusive.  You can only use one or
         the other in your aliases.
 
         Aliases expand Python variables just like system calls using ! or !!
         do: all expressions prefixed with '$' get expanded.  For details of
         the semantic rules, see PEP-215:
         http://www.python.org/peps/pep-0215.html.  This is the library used by
         IPython for variable expansion.  If you want to access a true shell
         variable, an extra $ is necessary to prevent its expansion by
         IPython::
 
           In [6]: alias show echo
           In [7]: PATH='A Python string'
           In [8]: show $PATH
           A Python string
           In [9]: show $$PATH
           /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
 
         You can use the alias facility to acess all of $PATH.  See the %rehash
         and %rehashx functions, which automatically create aliases for the
         contents of your $PATH.
 
         If called with no parameters, %alias prints the current alias table."""
 
         par = parameter_s.strip()
         if not par:
             aliases = sorted(self.shell.alias_manager.aliases)
             # stored = self.shell.db.get('stored_aliases', {} )
             # for k, v in stored:
             #     atab.append(k, v[0])
 
             print("Total number of aliases:", len(aliases))
             sys.stdout.flush()
             return aliases
 
         # Now try to define a new one
         try:
             alias,cmd = par.split(None, 1)
         except TypeError:
             print(oinspect.getdoc(self.alias))
             return
         
         try:
             self.shell.alias_manager.define_alias(alias, cmd)
         except AliasError as e:
             print(e)
     # end magic_alias
 
     @line_magic
     def unalias(self, parameter_s=''):
         """Remove an alias"""
 
         aname = parameter_s.strip()
         try:
             self.shell.alias_manager.undefine_alias(aname)
         except ValueError as e:
             print(e)
             return
         
         stored = self.shell.db.get('stored_aliases', {} )
         if aname in stored:
             print("Removing %stored alias",aname)
             del stored[aname]
             self.shell.db['stored_aliases'] = stored
 
     @line_magic
     def rehashx(self, parameter_s=''):
         """Update the alias table with all executable files in $PATH.
 
         This version explicitly checks that every entry in $PATH is a file
         with execute access (os.X_OK), so it is much slower than %rehash.
 
         Under Windows, it checks executability as a match against a
         '|'-separated string of extensions, stored in the IPython config
         variable win_exec_ext.  This defaults to 'exe|com|bat'.
 
         This function also resets the root module cache of module completer,
         used on slow filesystems.
         """
         from IPython.core.alias import InvalidAliasError
 
         # for the benefit of module completer in ipy_completers.py
         del self.shell.db['rootmodules_cache']
 
         path = [os.path.abspath(os.path.expanduser(p)) for p in
             os.environ.get('PATH','').split(os.pathsep)]
         path = filter(os.path.isdir,path)
 
         syscmdlist = []
         # Now define isexec in a cross platform manner.
         if os.name == 'posix':
             isexec = lambda fname:os.path.isfile(fname) and \
                      os.access(fname,os.X_OK)
         else:
             try:
                 winext = os.environ['pathext'].replace(';','|').replace('.','')
             except KeyError:
                 winext = 'exe|com|bat|py'
             if 'py' not in winext:
                 winext += '|py'
             execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
             isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
-        savedir = os.getcwdu()
+        savedir = py3compat.getcwd()
 
         # Now walk the paths looking for executables to alias.
         try:
             # write the whole loop for posix/Windows so we don't have an if in
             # the innermost part
             if os.name == 'posix':
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         if isexec(ff):
                             try:
                                 # Removes dots from the name since ipython
                                 # will assume names with dots to be python.
                                 if not self.shell.alias_manager.is_alias(ff):
                                     self.shell.alias_manager.define_alias(
                                         ff.replace('.',''), ff)
                             except InvalidAliasError:
                                 pass
                             else:
                                 syscmdlist.append(ff)
             else:
                 no_alias = Alias.blacklist
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):
                         base, ext = os.path.splitext(ff)
                         if isexec(ff) and base.lower() not in no_alias:
                             if ext.lower() == '.exe':
                                 ff = base
                                 try:
                                     # Removes dots from the name since ipython
                                     # will assume names with dots to be python.
                                     self.shell.alias_manager.define_alias(
                                         base.lower().replace('.',''), ff)
                                 except InvalidAliasError:
                                     pass
                                 syscmdlist.append(ff)
             self.shell.db['syscmdlist'] = syscmdlist
         finally:
             os.chdir(savedir)
 
     @skip_doctest
     @line_magic
     def pwd(self, parameter_s=''):
         """Return the current working directory path.
 
         Examples
         --------
         ::
 
           In [9]: pwd
           Out[9]: '/home/tsuser/sprint/ipython'
         """
-        return os.getcwdu()
+        return py3compat.getcwd()
 
     @skip_doctest
     @line_magic
     def cd(self, parameter_s=''):
         """Change the current working directory.
 
         This command automatically maintains an internal list of directories
         you visit during your IPython session, in the variable _dh. The
         command %dhist shows this history nicely formatted. You can also
         do 'cd -<tab>' to see directory history conveniently.
 
         Usage:
 
           cd 'dir': changes to directory 'dir'.
 
           cd -: changes to the last visited directory.
 
           cd -<n>: changes to the n-th directory in the directory history.
 
           cd --foo: change to directory that matches 'foo' in history
 
           cd -b <bookmark_name>: jump to a bookmark set by %bookmark
              (note: cd <bookmark_name> is enough if there is no
               directory <bookmark_name>, but a bookmark with the name exists.)
               'cd -b <tab>' allows you to tab-complete bookmark names.
 
         Options:
 
         -q: quiet.  Do not print the working directory after the cd command is
         executed.  By default IPython's cd command does print this directory,
         since the default prompts do not display path information.
 
         Note that !cd doesn't work for this purpose because the shell where
         !command runs is immediately discarded after executing 'command'.
 
         Examples
         --------
         ::
 
           In [10]: cd parent/child
           /home/tsuser/parent/child
         """
 
-        oldcwd = os.getcwdu()
+        oldcwd = py3compat.getcwd()
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
             nn = int(numcd.group(2))
             try:
                 ps = self.shell.user_ns['_dh'][nn]
             except IndexError:
                 print('The requested directory does not exist in history.')
                 return
             else:
                 opts = {}
         elif parameter_s.startswith('--'):
             ps = None
             fallback = None
             pat = parameter_s[2:]
             dh = self.shell.user_ns['_dh']
             # first search only by basename (last component)
             for ent in reversed(dh):
                 if pat in os.path.basename(ent) and os.path.isdir(ent):
                     ps = ent
                     break
 
                 if fallback is None and pat in ent and os.path.isdir(ent):
                     fallback = ent
 
             # if we have no last part match, pick the first full path match
             if ps is None:
                 ps = fallback
 
             if ps is None:
                 print("No matching entry in directory history")
                 return
             else:
                 opts = {}
 
 
         else:
             #turn all non-space-escaping backslashes to slashes,
             # for c:\windows\directory\names\
             parameter_s = re.sub(r'\\(?! )','/', parameter_s)
             opts,ps = self.parse_options(parameter_s,'qb',mode='string')
         # jump to previous
         if ps == '-':
             try:
                 ps = self.shell.user_ns['_dh'][-2]
             except IndexError:
                 raise UsageError('%cd -: No previous directory to change to.')
         # jump to bookmark if needed
         else:
             if not os.path.isdir(ps) or 'b' in opts:
                 bkms = self.shell.db.get('bookmarks', {})
 
                 if ps in bkms:
                     target = bkms[ps]
                     print('(bookmark:%s) -> %s' % (ps, target))
                     ps = target
                 else:
                     if 'b' in opts:
                         raise UsageError("Bookmark '%s' not found.  "
                               "Use '%%bookmark -l' to see your bookmarks." % ps)
 
         # strip extra quotes on Windows, because os.chdir doesn't like them
         ps = unquote_filename(ps)
         # at this point ps should point to the target dir
         if ps:
             try:
                 os.chdir(os.path.expanduser(ps))
                 if hasattr(self.shell, 'term_title') and self.shell.term_title:
                     set_term_title('IPython: ' + abbrev_cwd())
             except OSError:
                 print(sys.exc_info()[1])
             else:
-                cwd = os.getcwdu()
+                cwd = py3compat.getcwd()
                 dhist = self.shell.user_ns['_dh']
                 if oldcwd != cwd:
                     dhist.append(cwd)
                     self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
 
         else:
             os.chdir(self.shell.home_dir)
             if hasattr(self.shell, 'term_title') and self.shell.term_title:
                 set_term_title('IPython: ' + '~')
-            cwd = os.getcwdu()
+            cwd = py3compat.getcwd()
             dhist = self.shell.user_ns['_dh']
 
             if oldcwd != cwd:
                 dhist.append(cwd)
                 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and self.shell.user_ns['_dh']:
             print(self.shell.user_ns['_dh'][-1])
 
 
     @line_magic
     def env(self, parameter_s=''):
         """List environment variables."""
 
         return dict(os.environ)
 
     @line_magic
     def pushd(self, parameter_s=''):
         """Place the current dir on stack and change directory.
 
         Usage:\\
           %pushd ['dirname']
         """
 
         dir_s = self.shell.dir_stack
         tgt = os.path.expanduser(unquote_filename(parameter_s))
-        cwd = os.getcwdu().replace(self.shell.home_dir,'~')
+        cwd = py3compat.getcwd().replace(self.shell.home_dir,'~')
         if tgt:
             self.cd(parameter_s)
         dir_s.insert(0,cwd)
         return self.shell.magic('dirs')
 
     @line_magic
     def popd(self, parameter_s=''):
         """Change to directory popped off the top of the stack.
         """
         if not self.shell.dir_stack:
             raise UsageError("%popd on empty stack")
         top = self.shell.dir_stack.pop(0)
         self.cd(top)
         print("popd ->",top)
 
     @line_magic
     def dirs(self, parameter_s=''):
         """Return the current directory stack."""
 
         return self.shell.dir_stack
 
     @line_magic
     def dhist(self, parameter_s=''):
         """Print your history of visited directories.
 
         %dhist       -> print full history\\
         %dhist n     -> print last n entries only\\
         %dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
 
         This history is automatically maintained by the %cd command, and
         always available as the global list variable _dh. You can use %cd -<n>
         to go to directory number <n>.
 
         Note that most of time, you should view directory history by entering
         cd -<TAB>.
 
         """
 
         dh = self.shell.user_ns['_dh']
         if parameter_s:
             try:
                 args = map(int,parameter_s.split())
             except:
                 self.arg_err(self.dhist)
                 return
             if len(args) == 1:
                 ini,fin = max(len(dh)-(args[0]),0),len(dh)
             elif len(args) == 2:
                 ini,fin = args
                 fin = min(fin, len(dh))
             else:
                 self.arg_err(self.dhist)
                 return
         else:
             ini,fin = 0,len(dh)
         print('Directory history (kept in _dh)')
         for i in range(ini, fin):
             print("%d: %s" % (i, dh[i]))
 
     @skip_doctest
     @line_magic
     def sc(self, parameter_s=''):
         """Shell capture - run shell command and capture output (DEPRECATED use !).
 
         DEPRECATED. Suboptimal, retained for backwards compatibility.
 
         You should use the form 'var = !command' instead. Example:
 
          "%sc -l myfiles = ls ~" should now be written as
 
          "myfiles = !ls ~"
 
         myfiles.s, myfiles.l and myfiles.n still apply as documented
         below.
 
         --
         %sc [options] varname=command
 
         IPython will run the given command using commands.getoutput(), and
         will then update the user's interactive namespace with a variable
         called varname, containing the value of the call.  Your command can
         contain shell wildcards, pipes, etc.
 
         The '=' sign in the syntax is mandatory, and the variable name you
         supply must follow Python's standard conventions for valid names.
 
         (A special format without variable name exists for internal use)
 
         Options:
 
           -l: list output.  Split the output on newlines into a list before
           assigning it to the given variable.  By default the output is stored
           as a single string.
 
           -v: verbose.  Print the contents of the variable.
 
         In most cases you should not need to split as a list, because the
         returned value is a special type of string which can automatically
         provide its contents either as a list (split on newlines) or as a
         space-separated string.  These are convenient, respectively, either
         for sequential processing or to be passed to a shell command.
 
         For example::
 
             # Capture into variable a
             In [1]: sc a=ls *py
 
             # a is a string with embedded newlines
             In [2]: a
             Out[2]: 'setup.py\\nwin32_manual_post_install.py'
 
             # which can be seen as a list:
             In [3]: a.l
             Out[3]: ['setup.py', 'win32_manual_post_install.py']
 
             # or as a whitespace-separated string:
             In [4]: a.s
             Out[4]: 'setup.py win32_manual_post_install.py'
 
             # a.s is useful to pass as a single command line:
             In [5]: !wc -l $a.s
               146 setup.py
               130 win32_manual_post_install.py
               276 total
 
             # while the list form is useful to loop over:
             In [6]: for f in a.l:
               ...:      !wc -l $f
               ...:
             146 setup.py
             130 win32_manual_post_install.py
 
         Similarly, the lists returned by the -l option are also special, in
         the sense that you can equally invoke the .s attribute on them to
         automatically get a whitespace-separated string from their contents::
 
             In [7]: sc -l b=ls *py
 
             In [8]: b
             Out[8]: ['setup.py', 'win32_manual_post_install.py']
 
             In [9]: b.s
             Out[9]: 'setup.py win32_manual_post_install.py'
 
         In summary, both the lists and strings used for output capture have
         the following special attributes::
 
             .l (or .list) : value as list.
             .n (or .nlstr): value as newline-separated string.
             .s (or .spstr): value as space-separated string.
         """
 
         opts,args = self.parse_options(parameter_s, 'lv')
         # Try to get a variable name and command to run
         try:
             # the variable name must be obtained from the parse_options
             # output, which uses shlex.split to strip options out.
             var,_ = args.split('=', 1)
             var = var.strip()
             # But the command has to be extracted from the original input
             # parameter_s, not on what parse_options returns, to avoid the
             # quote stripping which shlex.split performs on it.
             _,cmd = parameter_s.split('=', 1)
         except ValueError:
             var,cmd = '',''
         # If all looks ok, proceed
         split = 'l' in opts
         out = self.shell.getoutput(cmd, split=split)
         if 'v' in opts:
             print('%s ==\n%s' % (var, pformat(out)))
         if var:
             self.shell.user_ns.update({var:out})
         else:
             return out
 
     @line_cell_magic
     def sx(self, line='', cell=None):
         """Shell execute - run shell command and capture output (!! is short-hand).
 
         %sx command
 
         IPython will run the given command using commands.getoutput(), and
         return the result formatted as a list (split on '\\n').  Since the
         output is _returned_, it will be stored in ipython's regular output
         cache Out[N] and in the '_N' automatic variables.
 
         Notes:
 
         1) If an input line begins with '!!', then %sx is automatically
         invoked.  That is, while::
 
           !ls
 
         causes ipython to simply issue system('ls'), typing::
 
           !!ls
 
         is a shorthand equivalent to::
 
           %sx ls
 
         2) %sx differs from %sc in that %sx automatically splits into a list,
         like '%sc -l'.  The reason for this is to make it as easy as possible
         to process line-oriented shell output via further python commands.
         %sc is meant to provide much finer control, but requires more
         typing.
 
         3) Just like %sc -l, this is a list with special attributes:
         ::
 
           .l (or .list) : value as list.
           .n (or .nlstr): value as newline-separated string.
           .s (or .spstr): value as whitespace-separated string.
 
         This is very useful when trying to use such lists as arguments to
         system commands."""
         
         if cell is None:
             # line magic
             return self.shell.getoutput(line)
         else:
             opts,args = self.parse_options(line, '', 'out=')
             output = self.shell.getoutput(cell)
             out_name = opts.get('out', opts.get('o'))
             if out_name:
                 self.shell.user_ns[out_name] = output
             else:
                 return output
 
     system = line_cell_magic('system')(sx)
     bang = cell_magic('!')(sx)
 
     @line_magic
     def bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.
 
         %bookmark <name>       - set bookmark to current dir
         %bookmark <name> <dir> - set bookmark to <dir>
         %bookmark -l           - list all bookmarks
         %bookmark -d <name>    - remove bookmark
         %bookmark -r           - remove all bookmarks
 
         You can later on access a bookmarked folder with::
 
           %cd -b <name>
 
         or simply '%cd <name>' if there is no directory called <name> AND
         there is such a bookmark defined.
 
         Your bookmarks persist through IPython sessions, but they are
         associated with each profile."""
 
         opts,args = self.parse_options(parameter_s,'drl',mode='list')
         if len(args) > 2:
             raise UsageError("%bookmark: too many arguments")
 
         bkms = self.shell.db.get('bookmarks',{})
 
         if 'd' in opts:
             try:
                 todel = args[0]
             except IndexError:
                 raise UsageError(
                     "%bookmark -d: must provide a bookmark to delete")
             else:
                 try:
                     del bkms[todel]
                 except KeyError:
                     raise UsageError(
                         "%%bookmark -d: Can't delete bookmark '%s'" % todel)
 
         elif 'r' in opts:
             bkms = {}
         elif 'l' in opts:
             bks = bkms.keys()
             bks.sort()
             if bks:
                 size = max(map(len, bks))
             else:
                 size = 0
             fmt = '%-'+str(size)+'s -> %s'
             print('Current bookmarks:')
             for bk in bks:
                 print(fmt % (bk, bkms[bk]))
         else:
             if not args:
                 raise UsageError("%bookmark: You must specify the bookmark name")
             elif len(args)==1:
-                bkms[args[0]] = os.getcwdu()
+                bkms[args[0]] = py3compat.getcwd()
             elif len(args)==2:
                 bkms[args[0]] = args[1]
         self.shell.db['bookmarks'] = bkms
 
     @line_magic
     def pycat(self, parameter_s=''):
         """Show a syntax-highlighted file through a pager.
 
         This magic is similar to the cat utility, but it will assume the file
         to be Python source and will show it with syntax highlighting.
 
         This magic command can either take a local filename, an url,
         an history range (see %history) or a macro as argument ::
 
         %pycat myscript.py
         %pycat 7-27
         %pycat myMacro
         %pycat http://www.example.com/myscript.py
         """
         if not parameter_s:
             raise UsageError('Missing filename, URL, input history range, '
                              'or macro.')
 
         try :
             cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
         except (ValueError, IOError):
             print("Error: no such file, variable, URL, history range or macro")
             return
 
         page.page(self.shell.pycolorize(source_to_unicode(cont)))
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument(
         '-a', '--append', action='store_true', default=False,
         help='Append contents of the cell to an existing file. '
              'The file will be created if it does not exist.'
     )
     @magic_arguments.argument(
         'filename', type=unicode_type,
         help='file to write'
     )
     @cell_magic
     def writefile(self, line, cell):
         """Write the contents of the cell to a file.
         
         The file will be overwritten unless the -a (--append) flag is specified.
         """
         args = magic_arguments.parse_argstring(self.writefile, line)
         filename = os.path.expanduser(unquote_filename(args.filename))
         
         if os.path.exists(filename):
             if args.append:
                 print("Appending to %s" % filename)
             else:
                 print("Overwriting %s" % filename)
         else:
             print("Writing %s" % filename)
         
         mode = 'a' if args.append else 'w'
         with io.open(filename, mode, encoding='utf-8') as f:
             f.write(cell)

IPython/core/profileapp.py

@@ -1,314 +1,315 b''
 # encoding: utf-8
 """
 An application for managing IPython profiles.
 
 To be invoked as the `ipython profile` subcommand.
 
 Authors:
 
 * Min RK
 
 """
 from __future__ import print_function
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 
 from IPython.config.application import Application
 from IPython.core.application import (
     BaseIPythonApplication, base_flags
 )
 from IPython.core.profiledir import ProfileDir
 from IPython.utils.importstring import import_item
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
+from IPython.utils import py3compat
 from IPython.utils.traitlets import Unicode, Bool, Dict
 
 #-----------------------------------------------------------------------------
 # Constants
 #-----------------------------------------------------------------------------
 
 create_help = """Create an IPython profile by name
 
 Create an ipython profile directory by its name or
 profile directory path. Profile directories contain
 configuration, log and security related files and are named
 using the convention 'profile_<name>'. By default they are
 located in your ipython directory. Once created, you will
 can edit the configuration files in the profile
 directory to configure IPython. Most users will create a
 profile directory by name,
 `ipython profile create myprofile`, which will put the directory
 in `<ipython_dir>/profile_myprofile`.
 """
 list_help = """List available IPython profiles
 
 List all available profiles, by profile location, that can
 be found in the current working directly or in the ipython
 directory. Profile directories are named using the convention
 'profile_<profile>'.
 """
 profile_help = """Manage IPython profiles
 
 Profile directories contain
 configuration, log and security related files and are named
 using the convention 'profile_<name>'. By default they are
 located in your ipython directory.  You can create profiles
 with `ipython profile create <name>`, or see the profiles you
 already have with `ipython profile list`
 
 To get started configuring IPython, simply do:
 
 $> ipython profile create
 
 and IPython will create the default profile in <ipython_dir>/profile_default,
 where you can edit ipython_config.py to start configuring IPython.
 
 """
 
 _list_examples = "ipython profile list  # list all profiles"
 
 _create_examples = """
 ipython profile create foo         # create profile foo w/ default config files
 ipython profile create foo --reset # restage default config files over current
 ipython profile create foo --parallel # also stage parallel config files
 """
 
 _main_examples = """
 ipython profile create -h  # show the help string for the create subcommand
 ipython profile list -h    # show the help string for the list subcommand
 
 ipython locate profile foo # print the path to the directory for profile 'foo'
 """
 
 #-----------------------------------------------------------------------------
 # Profile Application Class (for `ipython profile` subcommand)
 #-----------------------------------------------------------------------------
 
 
 def list_profiles_in(path):
     """list profiles in a given root directory"""
     files = os.listdir(path)
     profiles = []
     for f in files:
         try:
             full_path = os.path.join(path, f)
         except UnicodeError:
             continue
         if os.path.isdir(full_path) and f.startswith('profile_'):
             profiles.append(f.split('_',1)[-1])
     return profiles
 
 
 def list_bundled_profiles():
     """list profiles that are bundled with IPython."""
     path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
     files = os.listdir(path)
     profiles = []
     for profile in files:
         full_path = os.path.join(path, profile)
         if os.path.isdir(full_path) and profile != "__pycache__":
             profiles.append(profile)
     return profiles
 
 
 class ProfileLocate(BaseIPythonApplication):
     description = """print the path to an IPython profile dir"""
     
     def parse_command_line(self, argv=None):
         super(ProfileLocate, self).parse_command_line(argv)
         if self.extra_args:
             self.profile = self.extra_args[0]
     
     def start(self):
         print(self.profile_dir.location)
 
 
 class ProfileList(Application):
     name = u'ipython-profile'
     description = list_help
     examples = _list_examples
 
     aliases = Dict({
         'ipython-dir' : 'ProfileList.ipython_dir',
         'log-level' : 'Application.log_level',
     })
     flags = Dict(dict(
         debug = ({'Application' : {'log_level' : 0}},
             "Set Application.log_level to 0, maximizing log output."
         )
     ))
 
     ipython_dir = Unicode(get_ipython_dir(), config=True,
         help="""
         The name of the IPython directory. This directory is used for logging
         configuration (through profiles), history storage, etc. The default
         is usually $HOME/.ipython. This options can also be specified through
         the environment variable IPYTHONDIR.
         """
     )
 
 
     def _print_profiles(self, profiles):
         """print list of profiles, indented."""
         for profile in profiles:
             print('    %s' % profile)
 
     def list_profile_dirs(self):
         profiles = list_bundled_profiles()
         if profiles:
             print()
             print("Available profiles in IPython:")
             self._print_profiles(profiles)
             print()
             print("    The first request for a bundled profile will copy it")
             print("    into your IPython directory (%s)," % self.ipython_dir)
             print("    where you can customize it.")
         
         profiles = list_profiles_in(self.ipython_dir)
         if profiles:
             print()
             print("Available profiles in %s:" % self.ipython_dir)
             self._print_profiles(profiles)
         
-        profiles = list_profiles_in(os.getcwdu())
+        profiles = list_profiles_in(py3compat.getcwd())
         if profiles:
             print()
-            print("Available profiles in current directory (%s):" % os.getcwdu())
+            print("Available profiles in current directory (%s):" % py3compat.getcwd())
             self._print_profiles(profiles)
         
         print()
         print("To use any of the above profiles, start IPython with:")
         print("    ipython --profile=<name>")
         print()
 
     def start(self):
         self.list_profile_dirs()
 
 
 create_flags = {}
 create_flags.update(base_flags)
 # don't include '--init' flag, which implies running profile create in other apps
 create_flags.pop('init')
 create_flags['reset'] = ({'ProfileCreate': {'overwrite' : True}},
                         "reset config files in this profile to the defaults.")
 create_flags['parallel'] = ({'ProfileCreate': {'parallel' : True}},
                         "Include the config files for parallel "
                         "computing apps (ipengine, ipcontroller, etc.)")
 
 
 class ProfileCreate(BaseIPythonApplication):
     name = u'ipython-profile'
     description = create_help
     examples = _create_examples
     auto_create = Bool(True, config=False)
     def _log_format_default(self):
         return "[%(name)s] %(message)s"
 
     def _copy_config_files_default(self):
         return True
 
     parallel = Bool(False, config=True,
         help="whether to include parallel computing config files")
     def _parallel_changed(self, name, old, new):
         parallel_files = [   'ipcontroller_config.py',
                             'ipengine_config.py',
                             'ipcluster_config.py'
                         ]
         if new:
             for cf in parallel_files:
                 self.config_files.append(cf)
         else:
             for cf in parallel_files:
                 if cf in self.config_files:
                     self.config_files.remove(cf)
 
     def parse_command_line(self, argv):
         super(ProfileCreate, self).parse_command_line(argv)
         # accept positional arg as profile name
         if self.extra_args:
             self.profile = self.extra_args[0]
 
     flags = Dict(create_flags)
 
     classes = [ProfileDir]
     
     def _import_app(self, app_path):
         """import an app class"""
         app = None
         name = app_path.rsplit('.', 1)[-1]
         try:
             app = import_item(app_path)
-        except ImportError as e:
+        except ImportError:
             self.log.info("Couldn't import %s, config file will be excluded", name)
         except Exception:
             self.log.warn('Unexpected error importing %s', name, exc_info=True)
         return app
 
     def init_config_files(self):
         super(ProfileCreate, self).init_config_files()
         # use local imports, since these classes may import from here
         from IPython.terminal.ipapp import TerminalIPythonApp
         apps = [TerminalIPythonApp]
         for app_path in (
             'IPython.qt.console.qtconsoleapp.IPythonQtConsoleApp',
             'IPython.html.notebookapp.NotebookApp',
             'IPython.nbconvert.nbconvertapp.NbConvertApp',
         ):
             app = self._import_app(app_path)
             if app is not None:
                 apps.append(app)
         if self.parallel:
             from IPython.parallel.apps.ipcontrollerapp import IPControllerApp
             from IPython.parallel.apps.ipengineapp import IPEngineApp
             from IPython.parallel.apps.ipclusterapp import IPClusterStart
             from IPython.parallel.apps.iploggerapp import IPLoggerApp
             apps.extend([
                 IPControllerApp,
                 IPEngineApp,
                 IPClusterStart,
                 IPLoggerApp,
             ])
         for App in apps:
             app = App()
             app.config.update(self.config)
             app.log = self.log
             app.overwrite = self.overwrite
             app.copy_config_files=True
             app.profile = self.profile
             app.init_profile_dir()
             app.init_config_files()
 
     def stage_default_config_file(self):
         pass
 
 
 class ProfileApp(Application):
     name = u'ipython-profile'
     description = profile_help
     examples = _main_examples
 
     subcommands = Dict(dict(
         create = (ProfileCreate, ProfileCreate.description.splitlines()[0]),
         list = (ProfileList, ProfileList.description.splitlines()[0]),
         locate = (ProfileLocate, ProfileLocate.description.splitlines()[0]),
     ))
 
     def start(self):
         if self.subapp is None:
             print("No subcommand specified. Must specify one of: %s"%(self.subcommands.keys()))
             print()
             self.print_description()
             self.print_subcommands()
             self.exit(1)
         else:
             return self.subapp.start()

IPython/core/profiledir.py

@@ -1,273 +1,274 b''
 # encoding: utf-8
 """
 An object for managing IPython profile directories.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 * Min RK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import shutil
 import errno
 
 from IPython.config.configurable import LoggingConfigurable
 from IPython.utils.path import get_ipython_package_dir, expand_path
+from IPython.utils import py3compat
 from IPython.utils.traitlets import Unicode, Bool
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 
 #-----------------------------------------------------------------------------
 # Module errors
 #-----------------------------------------------------------------------------
 
 class ProfileDirError(Exception):
     pass
 
 
 #-----------------------------------------------------------------------------
 # Class for managing profile directories
 #-----------------------------------------------------------------------------
 
 class ProfileDir(LoggingConfigurable):
     """An object to manage the profile directory and its resources.
 
     The profile directory is used by all IPython applications, to manage
     configuration, logging and security.
 
     This object knows how to find, create and manage these directories. This
     should be used by any code that wants to handle profiles.
     """
 
     security_dir_name = Unicode('security')
     log_dir_name = Unicode('log')
     startup_dir_name = Unicode('startup')
     pid_dir_name = Unicode('pid')
     static_dir_name = Unicode('static')
     security_dir = Unicode(u'')
     log_dir = Unicode(u'')
     startup_dir = Unicode(u'')
     pid_dir = Unicode(u'')
     static_dir = Unicode(u'')
 
     location = Unicode(u'', config=True,
         help="""Set the profile location directly. This overrides the logic used by the
         `profile` option.""",
         )
 
     _location_isset = Bool(False) # flag for detecting multiply set location
 
     def _location_changed(self, name, old, new):
         if self._location_isset:
             raise RuntimeError("Cannot set profile location more than once.")
         self._location_isset = True
         if not os.path.isdir(new):
             os.makedirs(new)
         
         # ensure config files exist:
         self.security_dir = os.path.join(new, self.security_dir_name)
         self.log_dir = os.path.join(new, self.log_dir_name)
         self.startup_dir = os.path.join(new, self.startup_dir_name)
         self.pid_dir = os.path.join(new, self.pid_dir_name)
         self.static_dir = os.path.join(new, self.static_dir_name)
         self.check_dirs()
 
     def _log_dir_changed(self, name, old, new):
         self.check_log_dir()
     
     def _mkdir(self, path, mode=None):
         """ensure a directory exists at a given path
         
         This is a version of os.mkdir, with the following differences:
         
         - returns True if it created the directory, False otherwise
         - ignores EEXIST, protecting against race conditions where
           the dir may have been created in between the check and
           the creation
         - sets permissions if requested and the dir already exists
         """
         if os.path.exists(path):
             if mode and os.stat(path).st_mode != mode:
                 try:
                     os.chmod(path, mode)
                 except OSError:
                     self.log.warn(
                         "Could not set permissions on %s",
                         path
                     )
             return False
         try:
             if mode:
                 os.mkdir(path, mode)
             else:
                 os.mkdir(path)
         except OSError as e:
             if e.errno == errno.EEXIST:
                 return False
             else:
                 raise
         
         return True
 
     def check_log_dir(self):
         self._mkdir(self.log_dir)
 
     def _startup_dir_changed(self, name, old, new):
         self.check_startup_dir()
 
     def check_startup_dir(self):
         self._mkdir(self.startup_dir)
 
         readme = os.path.join(self.startup_dir, 'README')
         src = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'README_STARTUP')
 
         if not os.path.exists(src):
             self.log.warn("Could not copy README_STARTUP to startup dir. Source file %s does not exist.", src)
 
         if os.path.exists(src) and not os.path.exists(readme):
             shutil.copy(src, readme)
 
     def _security_dir_changed(self, name, old, new):
         self.check_security_dir()
 
     def check_security_dir(self):
         self._mkdir(self.security_dir, 0o40700)
 
     def _pid_dir_changed(self, name, old, new):
         self.check_pid_dir()
 
     def check_pid_dir(self):
         self._mkdir(self.pid_dir, 0o40700)
 
     def _static_dir_changed(self, name, old, new):
         self.check_startup_dir()
 
     def check_static_dir(self):
         self._mkdir(self.static_dir)
         custom = os.path.join(self.static_dir, 'custom')
         self._mkdir(custom)
         from IPython.html import DEFAULT_STATIC_FILES_PATH
         for fname in ('custom.js', 'custom.css'):
             src = os.path.join(DEFAULT_STATIC_FILES_PATH, 'custom', fname)
             dest = os.path.join(custom, fname)
             if not os.path.exists(src):
                 self.log.warn("Could not copy default file to static dir. Source file %s does not exist.", src)
                 continue
             if not os.path.exists(dest):
                 shutil.copy(src, dest)
 
     def check_dirs(self):
         self.check_security_dir()
         self.check_log_dir()
         self.check_pid_dir()
         self.check_startup_dir()
         self.check_static_dir()
 
     def copy_config_file(self, config_file, path=None, overwrite=False):
         """Copy a default config file into the active profile directory.
 
         Default configuration files are kept in :mod:`IPython.config.default`.
         This function moves these from that location to the working profile
         directory.
         """
         dst = os.path.join(self.location, config_file)
         if os.path.isfile(dst) and not overwrite:
             return False
         if path is None:
             path = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
         src = os.path.join(path, config_file)
         shutil.copy(src, dst)
         return True
 
     @classmethod
     def create_profile_dir(cls, profile_dir, config=None):
         """Create a new profile directory given a full path.
 
         Parameters
         ----------
         profile_dir : str
             The full path to the profile directory.  If it does exist, it will
             be used.  If not, it will be created.
         """
         return cls(location=profile_dir, config=config)
 
     @classmethod
     def create_profile_dir_by_name(cls, path, name=u'default', config=None):
         """Create a profile dir by profile name and path.
 
         Parameters
         ----------
         path : unicode
             The path (directory) to put the profile directory in.
         name : unicode
             The name of the profile.  The name of the profile directory will
             be "profile_<profile>".
         """
         if not os.path.isdir(path):
             raise ProfileDirError('Directory not found: %s' % path)
         profile_dir = os.path.join(path, u'profile_' + name)
         return cls(location=profile_dir, config=config)
 
     @classmethod
     def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None):
         """Find an existing profile dir by profile name, return its ProfileDir.
 
         This searches through a sequence of paths for a profile dir.  If it
         is not found, a :class:`ProfileDirError` exception will be raised.
 
         The search path algorithm is:
-        1. ``os.getcwdu()``
+        1. ``py3compat.getcwd()``
         2. ``ipython_dir``
 
         Parameters
         ----------
         ipython_dir : unicode or str
             The IPython directory to use.
         name : unicode or str
             The name of the profile.  The name of the profile directory
             will be "profile_<profile>".
         """
         dirname = u'profile_' + name
-        paths = [os.getcwdu(), ipython_dir]
+        paths = [py3compat.getcwd(), ipython_dir]
         for p in paths:
             profile_dir = os.path.join(p, dirname)
             if os.path.isdir(profile_dir):
                 return cls(location=profile_dir, config=config)
         else:
             raise ProfileDirError('Profile directory not found in paths: %s' % dirname)
 
     @classmethod
     def find_profile_dir(cls, profile_dir, config=None):
         """Find/create a profile dir and return its ProfileDir.
 
         This will create the profile directory if it doesn't exist.
 
         Parameters
         ----------
         profile_dir : unicode or str
             The path of the profile directory.  This is expanded using
             :func:`IPython.utils.genutils.expand_path`.
         """
         profile_dir = expand_path(profile_dir)
         if not os.path.isdir(profile_dir):
             raise ProfileDirError('Profile directory not found: %s' % profile_dir)
         return cls(location=profile_dir, config=config)
 
 

IPython/core/prompts.py

@@ -1,439 +1,439 b''
 # -*- coding: utf-8 -*-
 """Classes for handling input/output prompts.
 
 Authors:
 
 * Fernando Perez
 * Brian Granger
 * Thomas Kluyver
 """
 
 #-----------------------------------------------------------------------------
 #       Copyright (C) 2008-2011 The IPython Development Team
 #       Copyright (C) 2001-2007 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import socket
 import sys
 import time
 
 from string import Formatter
 
 from IPython.config.configurable import Configurable
 from IPython.core import release
 from IPython.utils import coloransi, py3compat
 from IPython.utils.traitlets import (Unicode, Instance, Dict, Bool, Int)
 
 #-----------------------------------------------------------------------------
 # Color schemes for prompts
 #-----------------------------------------------------------------------------
 
 InputColors = coloransi.InputTermColors  # just a shorthand
 Colors = coloransi.TermColors  # just a shorthand
 
 color_lists = dict(normal=Colors(), inp=InputColors(), nocolor=coloransi.NoColors())
 
 PColNoColors = coloransi.ColorScheme(
     'NoColor',
     in_prompt  = InputColors.NoColor,  # Input prompt
     in_number  = InputColors.NoColor,  # Input prompt number
     in_prompt2 = InputColors.NoColor, # Continuation prompt
     in_normal  = InputColors.NoColor,  # color off (usu. Colors.Normal)
 
     out_prompt = Colors.NoColor, # Output prompt
     out_number = Colors.NoColor, # Output prompt number
 
     normal = Colors.NoColor  # color off (usu. Colors.Normal)
     )
 
 # make some schemes as instances so we can copy them for modification easily:
 PColLinux =  coloransi.ColorScheme(
     'Linux',
     in_prompt  = InputColors.Green,
     in_number  = InputColors.LightGreen,
     in_prompt2 = InputColors.Green,
     in_normal  = InputColors.Normal,  # color off (usu. Colors.Normal)
 
     out_prompt = Colors.Red,
     out_number = Colors.LightRed,
 
     normal = Colors.Normal
     )
 
 # Slightly modified Linux for light backgrounds
 PColLightBG  = PColLinux.copy('LightBG')
 
 PColLightBG.colors.update(
     in_prompt  = InputColors.Blue,
     in_number  = InputColors.LightBlue,
     in_prompt2 = InputColors.Blue
 )
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 class LazyEvaluate(object):
     """This is used for formatting strings with values that need to be updated
     at that time, such as the current time or working directory."""
     def __init__(self, func, *args, **kwargs):
         self.func = func
         self.args = args
         self.kwargs = kwargs
     
     def __call__(self, **kwargs):
         self.kwargs.update(kwargs)
         return self.func(*self.args, **self.kwargs)
     
     def __str__(self):
         return str(self())
     
     def __unicode__(self):
         return py3compat.unicode_type(self())
     
     def __format__(self, format_spec):
         return format(self(), format_spec)
 
 def multiple_replace(dict, text):
     """ Replace in 'text' all occurences of any key in the given
     dictionary by its corresponding value.  Returns the new string."""
 
     # Function by Xavier Defrang, originally found at:
     # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81330
 
     # Create a regular expression  from the dictionary keys
     regex = re.compile("(%s)" % "|".join(map(re.escape, dict.keys())))
     # For each match, look-up corresponding value in dictionary
     return regex.sub(lambda mo: dict[mo.string[mo.start():mo.end()]], text)
 
 #-----------------------------------------------------------------------------
 # Special characters that can be used in prompt templates, mainly bash-like
 #-----------------------------------------------------------------------------
 
 # If $HOME isn't defined (Windows), make it an absurd string so that it can
 # never be expanded out into '~'.  Basically anything which can never be a
 # reasonable directory name will do, we just want the $HOME -> '~' operation
 # to become a no-op.  We pre-compute $HOME here so it's not done on every
 # prompt call.
 
 # FIXME:
 
 # - This should be turned into a class which does proper namespace management,
 # since the prompt specials need to be evaluated in a certain namespace.
 # Currently it's just globals, which need to be managed manually by code
 # below.
 
 # - I also need to split up the color schemes from the prompt specials
 # somehow.  I don't have a clean design for that quite yet.
 
 HOME = py3compat.str_to_unicode(os.environ.get("HOME","//////:::::ZZZZZ,,,~~~"))
 
 # This is needed on FreeBSD, and maybe other systems which symlink /home to
 # /usr/home, but retain the $HOME variable as pointing to /home
 HOME = os.path.realpath(HOME)
 
 # We precompute a few more strings here for the prompt_specials, which are
 # fixed once ipython starts.  This reduces the runtime overhead of computing
 # prompt strings.
 USER           = py3compat.str_to_unicode(os.environ.get("USER",''))
 HOSTNAME       = py3compat.str_to_unicode(socket.gethostname())
 HOSTNAME_SHORT = HOSTNAME.split(".")[0]
 ROOT_SYMBOL    = "#" if (os.name=='nt' or os.getuid()==0) else "$"
 
 prompt_abbreviations = {
     # Prompt/history count
     '%n' : '{color.number}' '{count}' '{color.prompt}',
     r'\#': '{color.number}' '{count}' '{color.prompt}',
     # Just the prompt counter number, WITHOUT any coloring wrappers, so users
     # can get numbers displayed in whatever color they want.
     r'\N': '{count}',
 
     # Prompt/history count, with the actual digits replaced by dots.  Used
     # mainly in continuation prompts (prompt_in2)
     r'\D': '{dots}',
 
     # Current time
     r'\T' : '{time}',
     # Current working directory
     r'\w': '{cwd}',
     # Basename of current working directory.
     # (use os.sep to make this portable across OSes)
     r'\W' : '{cwd_last}',
     # These X<N> are an extension to the normal bash prompts.  They return
     # N terms of the path, after replacing $HOME with '~'
     r'\X0': '{cwd_x[0]}',
     r'\X1': '{cwd_x[1]}',
     r'\X2': '{cwd_x[2]}',
     r'\X3': '{cwd_x[3]}',
     r'\X4': '{cwd_x[4]}',
     r'\X5': '{cwd_x[5]}',
     # Y<N> are similar to X<N>, but they show '~' if it's the directory
     # N+1 in the list.  Somewhat like %cN in tcsh.
     r'\Y0': '{cwd_y[0]}',
     r'\Y1': '{cwd_y[1]}',
     r'\Y2': '{cwd_y[2]}',
     r'\Y3': '{cwd_y[3]}',
     r'\Y4': '{cwd_y[4]}',
     r'\Y5': '{cwd_y[5]}',
     # Hostname up to first .
     r'\h': HOSTNAME_SHORT,
     # Full hostname
     r'\H': HOSTNAME,
     # Username of current user
     r'\u': USER,
     # Escaped '\'
     '\\\\': '\\',
     # Newline
     r'\n': '\n',
     # Carriage return
     r'\r': '\r',
     # Release version
     r'\v': release.version,
     # Root symbol ($ or #)
     r'\$': ROOT_SYMBOL,
     }
 
 #-----------------------------------------------------------------------------
 # More utilities
 #-----------------------------------------------------------------------------
 
 def cwd_filt(depth):
     """Return the last depth elements of the current working directory.
 
     $HOME is always replaced with '~'.
     If depth==0, the full path is returned."""
 
-    cwd = os.getcwdu().replace(HOME,"~")
+    cwd = py3compat.getcwd().replace(HOME,"~")
     out = os.sep.join(cwd.split(os.sep)[-depth:])
     return out or os.sep
 
 def cwd_filt2(depth):
     """Return the last depth elements of the current working directory.
 
     $HOME is always replaced with '~'.
     If depth==0, the full path is returned."""
 
-    full_cwd = os.getcwdu()
+    full_cwd = py3compat.getcwd()
     cwd = full_cwd.replace(HOME,"~").split(os.sep)
     if '~' in cwd and len(cwd) == depth+1:
         depth += 1
     drivepart = ''
     if sys.platform == 'win32' and len(cwd) > depth:
         drivepart = os.path.splitdrive(full_cwd)[0]
     out = drivepart + '/'.join(cwd[-depth:])
 
     return out or os.sep
 
 #-----------------------------------------------------------------------------
 # Prompt classes
 #-----------------------------------------------------------------------------
 
 lazily_evaluate = {'time': LazyEvaluate(time.strftime, "%H:%M:%S"),
-                   'cwd': LazyEvaluate(os.getcwdu),
-                   'cwd_last': LazyEvaluate(lambda: os.getcwdu().split(os.sep)[-1]),
-                   'cwd_x': [LazyEvaluate(lambda: os.getcwdu().replace(HOME,"~"))] +\
+                   'cwd': LazyEvaluate(py3compat.getcwd),
+                   'cwd_last': LazyEvaluate(lambda: py3compat.getcwd().split(os.sep)[-1]),
+                   'cwd_x': [LazyEvaluate(lambda: py3compat.getcwd().replace(HOME,"~"))] +\
                             [LazyEvaluate(cwd_filt, x) for x in range(1,6)],
                    'cwd_y': [LazyEvaluate(cwd_filt2, x) for x in range(6)]
                    }
 
 def _lenlastline(s):
     """Get the length of the last line. More intelligent than
     len(s.splitlines()[-1]).
     """
     if not s or s.endswith(('\n', '\r')):
         return 0
     return len(s.splitlines()[-1])
 
 
 class UserNSFormatter(Formatter):
     """A Formatter that falls back on a shell's user_ns and __builtins__ for name resolution"""
     def __init__(self, shell):
         self.shell = shell
 
     def get_value(self, key, args, kwargs):
         # try regular formatting first:
         try:
             return Formatter.get_value(self, key, args, kwargs)
         except Exception:
             pass
         # next, look in user_ns and builtins:
         for container in (self.shell.user_ns, __builtins__):
             if key in container:
                 return container[key]
         # nothing found, put error message in its place
         return "<ERROR: '%s' not found>" % key
 
 
 class PromptManager(Configurable):
     """This is the primary interface for producing IPython's prompts."""
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     
     color_scheme_table = Instance(coloransi.ColorSchemeTable)
     color_scheme = Unicode('Linux', config=True)
     def _color_scheme_changed(self, name, new_value):
         self.color_scheme_table.set_active_scheme(new_value)
         for pname in ['in', 'in2', 'out', 'rewrite']:
             # We need to recalculate the number of invisible characters
             self.update_prompt(pname)
     
     lazy_evaluate_fields = Dict(help="""
         This maps field names used in the prompt templates to functions which
         will be called when the prompt is rendered. This allows us to include
         things like the current time in the prompts. Functions are only called
         if they are used in the prompt.
         """)
     def _lazy_evaluate_fields_default(self): return lazily_evaluate.copy()
     
     in_template = Unicode('In [\\#]: ', config=True,
         help="Input prompt.  '\\#' will be transformed to the prompt number")
     in2_template = Unicode('   .\\D.: ', config=True,
         help="Continuation prompt.")
     out_template = Unicode('Out[\\#]: ', config=True,
         help="Output prompt. '\\#' will be transformed to the prompt number")
     
     justify = Bool(True, config=True, help="""
         If True (default), each prompt will be right-aligned with the
         preceding one.
         """)
     
     # We actually store the expanded templates here:
     templates = Dict()
     
     # The number of characters in the last prompt rendered, not including
     # colour characters.
     width = Int()
     txtwidth = Int()   # Not including right-justification
     
     # The number of characters in each prompt which don't contribute to width
     invisible_chars = Dict()
     def _invisible_chars_default(self):
         return {'in': 0, 'in2': 0, 'out': 0, 'rewrite':0}
     
     def __init__(self, shell, **kwargs):
         super(PromptManager, self).__init__(shell=shell, **kwargs)
         
         # Prepare colour scheme table
         self.color_scheme_table = coloransi.ColorSchemeTable([PColNoColors,
                                     PColLinux, PColLightBG], self.color_scheme)
         
         self._formatter = UserNSFormatter(shell)
         # Prepare templates & numbers of invisible characters
         self.update_prompt('in', self.in_template)
         self.update_prompt('in2', self.in2_template)
         self.update_prompt('out', self.out_template)
         self.update_prompt('rewrite')
         self.on_trait_change(self._update_prompt_trait, ['in_template',
                             'in2_template', 'out_template'])
     
     def update_prompt(self, name, new_template=None):
         """This is called when a prompt template is updated. It processes
         abbreviations used in the prompt template (like \#) and calculates how
         many invisible characters (ANSI colour escapes) the resulting prompt
         contains.
         
         It is also called for each prompt on changing the colour scheme. In both
         cases, traitlets should take care of calling this automatically.
         """
         if new_template is not None:
             self.templates[name] = multiple_replace(prompt_abbreviations, new_template)
         # We count invisible characters (colour escapes) on the last line of the
         # prompt, to calculate the width for lining up subsequent prompts.
         invis_chars = _lenlastline(self._render(name, color=True)) - \
                         _lenlastline(self._render(name, color=False))
         self.invisible_chars[name] = invis_chars
     
     def _update_prompt_trait(self, traitname, new_template):
         name = traitname[:-9]   # Cut off '_template'
         self.update_prompt(name, new_template)
     
     def _render(self, name, color=True, **kwargs):
         """Render but don't justify, or update the width or txtwidth attributes.
         """
         if name == 'rewrite':
             return self._render_rewrite(color=color)
         
         if color:
             scheme = self.color_scheme_table.active_colors
             if name=='out':
                 colors = color_lists['normal']
                 colors.number, colors.prompt, colors.normal = \
                         scheme.out_number, scheme.out_prompt, scheme.normal
             else:
                 colors = color_lists['inp']
                 colors.number, colors.prompt, colors.normal = \
                         scheme.in_number, scheme.in_prompt, scheme.in_normal
                 if name=='in2':
                     colors.prompt = scheme.in_prompt2
         else:
             # No color
             colors = color_lists['nocolor']
             colors.number, colors.prompt, colors.normal = '', '', ''
         
         count = self.shell.execution_count    # Shorthand
         # Build the dictionary to be passed to string formatting
         fmtargs = dict(color=colors, count=count,
                         dots="."*len(str(count)),
                         width=self.width, txtwidth=self.txtwidth )
         fmtargs.update(self.lazy_evaluate_fields)
         fmtargs.update(kwargs)
         
         # Prepare the prompt
         prompt = colors.prompt + self.templates[name] + colors.normal
         
         # Fill in required fields
         return self._formatter.format(prompt, **fmtargs)
     
     def _render_rewrite(self, color=True):
         """Render the ---> rewrite prompt."""
         if color:
             scheme = self.color_scheme_table.active_colors
             # We need a non-input version of these escapes
             color_prompt = scheme.in_prompt.replace("\001","").replace("\002","")
             color_normal = scheme.normal
         else:
             color_prompt, color_normal = '', ''
 
         return color_prompt + "-> ".rjust(self.txtwidth, "-") + color_normal
     
     def render(self, name, color=True, just=None, **kwargs):
         """
         Render the selected prompt.
         
         Parameters
         ----------
         name : str
           Which prompt to render. One of 'in', 'in2', 'out', 'rewrite'
         color : bool
           If True (default), include ANSI escape sequences for a coloured prompt.
         just : bool
           If True, justify the prompt to the width of the last prompt. The
           default is stored in self.justify.
         **kwargs :
           Additional arguments will be passed to the string formatting operation,
           so they can override the values that would otherwise fill in the
           template.
         
         Returns
         -------
         A string containing the rendered prompt.
         """
         res = self._render(name, color=color, **kwargs)
         
         # Handle justification of prompt
         invis_chars = self.invisible_chars[name] if color else 0
         self.txtwidth = _lenlastline(res) - invis_chars
         just = self.justify if (just is None) else just
         # If the prompt spans more than one line, don't try to justify it:
         if just and name != 'in' and ('\n' not in res) and ('\r' not in res):
             res = res.rjust(self.width + invis_chars)
         self.width = _lenlastline(res) - invis_chars
         return res

IPython/core/tests/test_application.py

@@ -1,50 +1,50 b''
 # coding: utf-8
 """Tests for IPython.core.application"""
 
 import os
 import tempfile
 
 from IPython.core.application import BaseIPythonApplication
 from IPython.testing import decorators as dec
 from IPython.utils import py3compat
 
 @dec.onlyif_unicode_paths
 def test_unicode_cwd():
     """Check that IPython starts with non-ascii characters in the path."""
     wd = tempfile.mkdtemp(suffix=u"€")
     
-    old_wd = os.getcwdu()
+    old_wd = py3compat.getcwd()
     os.chdir(wd)
-    #raise Exception(repr(os.getcwdu()))
+    #raise Exception(repr(py3compat.getcwd()))
     try:
         app = BaseIPythonApplication()
         # The lines below are copied from Application.initialize()
         app.init_profile_dir()
         app.init_config_files()
         app.load_config_file(suppress_errors=False)
     finally:
         os.chdir(old_wd)
 
 @dec.onlyif_unicode_paths
 def test_unicode_ipdir():
     """Check that IPython starts with non-ascii characters in the IP dir."""
     ipdir = tempfile.mkdtemp(suffix=u"€")
     
     # Create the config file, so it tries to load it.
     with open(os.path.join(ipdir, 'ipython_config.py'), "w") as f:
         pass
     
     old_ipdir1 = os.environ.pop("IPYTHONDIR", None)
     old_ipdir2 = os.environ.pop("IPYTHON_DIR", None)
     os.environ["IPYTHONDIR"] = py3compat.unicode_to_str(ipdir, "utf-8")
     try:
         app = BaseIPythonApplication()
         # The lines below are copied from Application.initialize()
         app.init_profile_dir()
         app.init_config_files()
         app.load_config_file(suppress_errors=False)
     finally:
         if old_ipdir1:
             os.environ["IPYTHONDIR"] = old_ipdir1
         if old_ipdir2:
             os.environ["IPYTHONDIR"] = old_ipdir2

IPython/core/tests/test_completer.py

@@ -1,393 +1,394 b''
 """Tests for the IPython tab-completion machinery.
 """
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 # stdlib
 import os
 import sys
 import unittest
 
 # third party
 import nose.tools as nt
 
 # our own packages
 from IPython.config.loader import Config
 from IPython.core import completer
 from IPython.external.decorators import knownfailureif
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.utils.generics import complete_object
+from IPython.utils import py3compat
 from IPython.utils.py3compat import string_types, unicode_type
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 def test_protect_filename():
     pairs = [ ('abc','abc'),
               (' abc',r'\ abc'),
               ('a bc',r'a\ bc'),
               ('a  bc',r'a\ \ bc'),
               ('  bc',r'\ \ bc'),
               ]
     # On posix, we also protect parens and other special characters
     if sys.platform != 'win32':
         pairs.extend( [('a(bc',r'a\(bc'),
                        ('a)bc',r'a\)bc'),
                        ('a( )bc',r'a\(\ \)bc'),
                        ('a[1]bc', r'a\[1\]bc'),
                        ('a{1}bc', r'a\{1\}bc'),
                        ('a#bc', r'a\#bc'),
                        ('a?bc', r'a\?bc'),
                        ('a=bc', r'a\=bc'),
                        ('a\\bc', r'a\\bc'),
                        ('a|bc', r'a\|bc'),
                        ('a;bc', r'a\;bc'),
                        ('a:bc', r'a\:bc'),
                        ("a'bc", r"a\'bc"),
                        ('a*bc', r'a\*bc'),
                        ('a"bc', r'a\"bc'),
                        ('a^bc', r'a\^bc'),
                        ('a&bc', r'a\&bc'),
                        ] )
     # run the actual tests
     for s1, s2 in pairs:
         s1p = completer.protect_filename(s1)
         nt.assert_equal(s1p, s2)
 
 
 def check_line_split(splitter, test_specs):
     for part1, part2, split in test_specs:
         cursor_pos = len(part1)
         line = part1+part2
         out = splitter.split_line(line, cursor_pos)
         nt.assert_equal(out, split)
 
 
 def test_line_split():
     """Basic line splitter test with default specs."""
     sp = completer.CompletionSplitter()
     # The format of the test specs is: part1, part2, expected answer.  Parts 1
     # and 2 are joined into the 'line' sent to the splitter, as if the cursor
     # was at the end of part1.  So an empty part2 represents someone hitting
     # tab at the end of the line, the most common case.
     t = [('run some/scrip', '', 'some/scrip'),
          ('run scripts/er', 'ror.py foo', 'scripts/er'),
          ('echo $HOM', '', 'HOM'),
          ('print sys.pa', '', 'sys.pa'),
          ('print(sys.pa', '', 'sys.pa'),
          ("execfile('scripts/er", '', 'scripts/er'),
          ('a[x.', '', 'x.'),
          ('a[x.', 'y', 'x.'),
          ('cd "some_file/', '', 'some_file/'),
          ]
     check_line_split(sp, t)
     # Ensure splitting works OK with unicode by re-running the tests with
     # all inputs turned into unicode
     check_line_split(sp, [ map(unicode_type, p) for p in t] )
 
 
 def test_custom_completion_error():
     """Test that errors from custom attribute completers are silenced."""
     ip = get_ipython()
     class A(object): pass
     ip.user_ns['a'] = A()
     
     @complete_object.when_type(A)
     def complete_A(a, existing_completions):
         raise TypeError("this should be silenced")
     
     ip.complete("a.")
 
 
 def test_unicode_completions():
     ip = get_ipython()
     # Some strings that trigger different types of completion.  Check them both
     # in str and unicode forms
     s = ['ru', '%ru', 'cd /', 'floa', 'float(x)/']
     for t in s + list(map(unicode_type, s)):
         # We don't need to check exact completion values (they may change
         # depending on the state of the namespace, but at least no exceptions
         # should be thrown and the return value should be a pair of text, list
         # values.
         text, matches = ip.complete(t)
         nt.assert_true(isinstance(text, string_types))
         nt.assert_true(isinstance(matches, list))
 
 
 class CompletionSplitterTestCase(unittest.TestCase):
     def setUp(self):
         self.sp = completer.CompletionSplitter()
 
     def test_delim_setting(self):
         self.sp.delims = ' '
         nt.assert_equal(self.sp.delims, ' ')
         nt.assert_equal(self.sp._delim_expr, '[\ ]')
 
     def test_spaces(self):
         """Test with only spaces as split chars."""
         self.sp.delims = ' '
         t = [('foo', '', 'foo'),
              ('run foo', '', 'foo'),
              ('run foo', 'bar', 'foo'),
              ]
         check_line_split(self.sp, t)
 
 
 def test_has_open_quotes1():
     for s in ["'", "'''", "'hi' '"]:
         nt.assert_equal(completer.has_open_quotes(s), "'")
 
 
 def test_has_open_quotes2():
     for s in ['"', '"""', '"hi" "']:
         nt.assert_equal(completer.has_open_quotes(s), '"')
 
 
 def test_has_open_quotes3():
     for s in ["''", "''' '''", "'hi' 'ipython'"]:
         nt.assert_false(completer.has_open_quotes(s))
 
 
 def test_has_open_quotes4():
     for s in ['""', '""" """', '"hi" "ipython"']:
         nt.assert_false(completer.has_open_quotes(s))
 
 
 @knownfailureif(sys.platform == 'win32', "abspath completions fail on Windows")
 def test_abspath_file_completions():
     ip = get_ipython()
     with TemporaryDirectory() as tmpdir:
         prefix = os.path.join(tmpdir, 'foo')
         suffixes = ['1', '2']
         names = [prefix+s for s in suffixes]
         for n in names:
             open(n, 'w').close()
 
         # Check simple completion
         c = ip.complete(prefix)[1]
         nt.assert_equal(c, names)
 
         # Now check with a function call
         cmd = 'a = f("%s' % prefix
         c = ip.complete(prefix, cmd)[1]
         comp = [prefix+s for s in suffixes]
         nt.assert_equal(c, comp)
 
 
 def test_local_file_completions():
     ip = get_ipython()
-    cwd = os.getcwdu()
+    cwd = py3compat.getcwd()
     try:
         with TemporaryDirectory() as tmpdir:
             os.chdir(tmpdir)
             prefix = './foo'
             suffixes = ['1', '2']
             names = [prefix+s for s in suffixes]
             for n in names:
                 open(n, 'w').close()
 
             # Check simple completion
             c = ip.complete(prefix)[1]
             nt.assert_equal(c, names)
 
             # Now check with a function call
             cmd = 'a = f("%s' % prefix
             c = ip.complete(prefix, cmd)[1]
             comp = [prefix+s for s in suffixes]
             nt.assert_equal(c, comp)
     finally:
         # prevent failures from making chdir stick
         os.chdir(cwd)
 
 
 def test_greedy_completions():
     ip = get_ipython()
     greedy_original = ip.Completer.greedy
     try:
         ip.Completer.greedy = False
         ip.ex('a=list(range(5))')
         _,c = ip.complete('.',line='a[0].')
         nt.assert_false('a[0].real' in c,
                         "Shouldn't have completed on a[0]: %s"%c)
         ip.Completer.greedy = True
         _,c = ip.complete('.',line='a[0].')
         nt.assert_true('a[0].real' in c, "Should have completed on a[0]: %s"%c)
     finally:
         ip.Completer.greedy = greedy_original
 
 
 def test_omit__names():
     # also happens to test IPCompleter as a configurable
     ip = get_ipython()
     ip._hidden_attr = 1
     c = ip.Completer
     ip.ex('ip=get_ipython()')
     cfg = Config()
     cfg.IPCompleter.omit__names = 0
     c.update_config(cfg)
     s,matches = c.complete('ip.')
     nt.assert_in('ip.__str__', matches)
     nt.assert_in('ip._hidden_attr', matches)
     cfg.IPCompleter.omit__names = 1
     c.update_config(cfg)
     s,matches = c.complete('ip.')
     nt.assert_not_in('ip.__str__', matches)
     nt.assert_in('ip._hidden_attr', matches)
     cfg.IPCompleter.omit__names = 2
     c.update_config(cfg)
     s,matches = c.complete('ip.')
     nt.assert_not_in('ip.__str__', matches)
     nt.assert_not_in('ip._hidden_attr', matches)
     del ip._hidden_attr
 
 
 def test_limit_to__all__False_ok():
     ip = get_ipython()
     c = ip.Completer
     ip.ex('class D: x=24')
     ip.ex('d=D()')
     cfg = Config()
     cfg.IPCompleter.limit_to__all__ = False
     c.update_config(cfg)
     s, matches = c.complete('d.')
     nt.assert_in('d.x', matches)
 
 
 def test_limit_to__all__True_ok():
     ip = get_ipython()
     c = ip.Completer
     ip.ex('class D: x=24')
     ip.ex('d=D()')
     ip.ex("d.__all__=['z']")
     cfg = Config()
     cfg.IPCompleter.limit_to__all__ = True
     c.update_config(cfg)
     s, matches = c.complete('d.')
     nt.assert_in('d.z', matches)
     nt.assert_not_in('d.x', matches)
 
 
 def test_get__all__entries_ok():
     class A(object):
         __all__ = ['x', 1]
     words = completer.get__all__entries(A())
     nt.assert_equal(words, ['x'])
 
 
 def test_get__all__entries_no__all__ok():
     class A(object):
         pass
     words = completer.get__all__entries(A())
     nt.assert_equal(words, [])
 
 
 def test_func_kw_completions():
     ip = get_ipython()
     c = ip.Completer
     ip.ex('def myfunc(a=1,b=2): return a+b')
     s, matches = c.complete(None, 'myfunc(1,b')
     nt.assert_in('b=', matches)
     # Simulate completing with cursor right after b (pos==10):
     s, matches = c.complete(None, 'myfunc(1,b)', 10)
     nt.assert_in('b=', matches)
     s, matches = c.complete(None, 'myfunc(a="escaped\\")string",b')
     nt.assert_in('b=', matches)
     #builtin function
     s, matches = c.complete(None, 'min(k, k')
     nt.assert_in('key=', matches)
 
 
 def test_default_arguments_from_docstring():
     doc = min.__doc__
     ip = get_ipython()
     c = ip.Completer
     kwd = c._default_arguments_from_docstring(
         'min(iterable[, key=func]) -> value')
     nt.assert_equal(kwd, ['key'])
     #with cython type etc
     kwd = c._default_arguments_from_docstring(
         'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)\n')
     nt.assert_equal(kwd, ['ncall', 'resume', 'nsplit'])
     #white spaces
     kwd = c._default_arguments_from_docstring(
         '\n Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)\n')
     nt.assert_equal(kwd, ['ncall', 'resume', 'nsplit'])
 
 def test_line_magics():
     ip = get_ipython()
     c = ip.Completer
     s, matches = c.complete(None, 'lsmag')
     nt.assert_in('%lsmagic', matches)
     s, matches = c.complete(None, '%lsmag')
     nt.assert_in('%lsmagic', matches)
 
 
 def test_cell_magics():
     from IPython.core.magic import register_cell_magic
 
     @register_cell_magic
     def _foo_cellm(line, cell):
         pass
     
     ip = get_ipython()
     c = ip.Completer
 
     s, matches = c.complete(None, '_foo_ce')
     nt.assert_in('%%_foo_cellm', matches)
     s, matches = c.complete(None, '%%_foo_ce')
     nt.assert_in('%%_foo_cellm', matches)
 
 
 def test_line_cell_magics():
     from IPython.core.magic import register_line_cell_magic
 
     @register_line_cell_magic
     def _bar_cellm(line, cell):
         pass
     
     ip = get_ipython()
     c = ip.Completer
 
     # The policy here is trickier, see comments in completion code.  The
     # returned values depend on whether the user passes %% or not explicitly,
     # and this will show a difference if the same name is both a line and cell
     # magic.
     s, matches = c.complete(None, '_bar_ce')
     nt.assert_in('%_bar_cellm', matches)
     nt.assert_in('%%_bar_cellm', matches)
     s, matches = c.complete(None, '%_bar_ce')
     nt.assert_in('%_bar_cellm', matches)
     nt.assert_in('%%_bar_cellm', matches)
     s, matches = c.complete(None, '%%_bar_ce')
     nt.assert_not_in('%_bar_cellm', matches)
     nt.assert_in('%%_bar_cellm', matches)
 
 
 def test_magic_completion_order():
 
     ip = get_ipython()
     c = ip.Completer
 
     # Test ordering of magics and non-magics with the same name
     # We want the non-magic first
 
     # Before importing matplotlib, there should only be one option:
 
     text, matches = c.complete('mat')
     nt.assert_equal(matches, ["%matplotlib"])
 
 
     ip.run_cell("matplotlib = 1")  # introduce name into namespace
 
     # After the import, there should be two options, ordered like this:
     text, matches = c.complete('mat')
     nt.assert_equal(matches, ["matplotlib", "%matplotlib"])
 
 
     ip.run_cell("timeit = 1")  # define a user variable called 'timeit'
 
     # Order of user variable and line and cell magics with same name:
     text, matches = c.complete('timeit')
     nt.assert_equal(matches, ["timeit", "%timeit","%%timeit"])
 

IPython/core/tests/test_completerlib.py

@@ -1,120 +1,121 b''
 # -*- coding: utf-8 -*-
 """Tests for completerlib.
 
 """
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import shutil
 import sys
 import tempfile
 import unittest
 from os.path import join
 
 from IPython.core.completerlib import magic_run_completer, module_completion
+from IPython.utils import py3compat
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.testing.decorators import onlyif_unicode_paths
 
 
 class MockEvent(object):
     def __init__(self, line):
         self.line = line
 
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 class Test_magic_run_completer(unittest.TestCase):
     def setUp(self):
         self.BASETESTDIR = tempfile.mkdtemp()
         for fil in [u"aao.py", u"a.py", u"b.py"]:
             with open(join(self.BASETESTDIR, fil), "w") as sfile:
                 sfile.write("pass\n")
-        self.oldpath = os.getcwdu()
+        self.oldpath = py3compat.getcwd()
         os.chdir(self.BASETESTDIR)
 
     def tearDown(self):
         os.chdir(self.oldpath)
         shutil.rmtree(self.BASETESTDIR)
 
     def test_1(self):
         """Test magic_run_completer, should match two alterntives
         """
         event = MockEvent(u"%run a")
         mockself = None
         match = set(magic_run_completer(mockself, event))
         self.assertEqual(match, set([u"a.py", u"aao.py"]))
 
     def test_2(self):
         """Test magic_run_completer, should match one alterntive
         """
         event = MockEvent(u"%run aa")
         mockself = None
         match = set(magic_run_completer(mockself, event))
         self.assertEqual(match, set([u"aao.py"]))
 
     def test_3(self):
         """Test magic_run_completer with unterminated " """
         event = MockEvent(u'%run "a')
         mockself = None
         match = set(magic_run_completer(mockself, event))
         self.assertEqual(match, set([u"a.py", u"aao.py"]))
 
     def test_import_invalid_module(self):
         """Testing of issue https://github.com/ipython/ipython/issues/1107"""
         invalid_module_names = set(['foo-bar', 'foo:bar', '10foo'])
         valid_module_names = set(['foobar'])
         with TemporaryDirectory() as tmpdir:
             sys.path.insert( 0, tmpdir )
             for name in invalid_module_names | valid_module_names:
                 filename = os.path.join(tmpdir, name + '.py')
                 open(filename, 'w').close()
 
             s = set( module_completion('import foo') )
             intersection = s.intersection(invalid_module_names)
             self.assertFalse(intersection, intersection)
             
             assert valid_module_names.issubset(s), valid_module_names.intersection(s)
 
 class Test_magic_run_completer_nonascii(unittest.TestCase):
     @onlyif_unicode_paths
     def setUp(self):
         self.BASETESTDIR = tempfile.mkdtemp()
         for fil in [u"aaø.py", u"a.py", u"b.py"]:
             with open(join(self.BASETESTDIR, fil), "w") as sfile:
                 sfile.write("pass\n")
-        self.oldpath = os.getcwdu()
+        self.oldpath = py3compat.getcwd()
         os.chdir(self.BASETESTDIR)
 
     def tearDown(self):
         os.chdir(self.oldpath)
         shutil.rmtree(self.BASETESTDIR)
 
     @onlyif_unicode_paths
     def test_1(self):
         """Test magic_run_completer, should match two alterntives
         """
         event = MockEvent(u"%run a")
         mockself = None
         match = set(magic_run_completer(mockself, event))
         self.assertEqual(match, set([u"a.py", u"aaø.py"]))
 
     @onlyif_unicode_paths
     def test_2(self):
         """Test magic_run_completer, should match one alterntive
         """
         event = MockEvent(u"%run aa")
         mockself = None
         match = set(magic_run_completer(mockself, event))
         self.assertEqual(match, set([u"aaø.py"]))
 
     @onlyif_unicode_paths
     def test_3(self):
         """Test magic_run_completer with unterminated " """
         event = MockEvent(u'%run "a')
         mockself = None
         match = set(magic_run_completer(mockself, event))
         self.assertEqual(match, set([u"a.py", u"aaø.py"]))

IPython/core/tests/test_interactiveshell.py

@@ -1,676 +1,677 b''
 # -*- coding: utf-8 -*-
 """Tests for the key interactiveshell module.
 
 Historically the main classes in interactiveshell have been under-tested.  This
 module should grow as many single-method tests as possible to trap many of the
 recurring bugs we seem to encounter with high-level interaction.
 
 Authors
 -------
 * Fernando Perez
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # stdlib
 import ast
 import os
 import signal
 import shutil
 import sys
 import tempfile
 import unittest
 from os.path import join
 
 # third-party
 import nose.tools as nt
 
 # Our own
 from IPython.testing.decorators import skipif, skip_win32, onlyif_unicode_paths
 from IPython.testing import tools as tt
 from IPython.utils import io
+from IPython.utils import py3compat
 from IPython.utils.py3compat import unicode_type, PY3
 
 if PY3:
     from io import StringIO
 else:
     from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 # This is used by every single test, no point repeating it ad nauseam
 ip = get_ipython()
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 class InteractiveShellTestCase(unittest.TestCase):
     def test_naked_string_cells(self):
         """Test that cells with only naked strings are fully executed"""
         # First, single-line inputs
         ip.run_cell('"a"\n')
         self.assertEqual(ip.user_ns['_'], 'a')
         # And also multi-line cells
         ip.run_cell('"""a\nb"""\n')
         self.assertEqual(ip.user_ns['_'], 'a\nb')
 
     def test_run_empty_cell(self):
         """Just make sure we don't get a horrible error with a blank
         cell of input. Yes, I did overlook that."""
         old_xc = ip.execution_count
         ip.run_cell('')
         self.assertEqual(ip.execution_count, old_xc)
 
     def test_run_cell_multiline(self):
         """Multi-block, multi-line cells must execute correctly.
         """
         src = '\n'.join(["x=1",
                          "y=2",
                          "if 1:",
                          "    x += 1",
                          "    y += 1",])
         ip.run_cell(src)
         self.assertEqual(ip.user_ns['x'], 2)
         self.assertEqual(ip.user_ns['y'], 3)
 
     def test_multiline_string_cells(self):
         "Code sprinkled with multiline strings should execute (GH-306)"
         ip.run_cell('tmp=0')
         self.assertEqual(ip.user_ns['tmp'], 0)
         ip.run_cell('tmp=1;"""a\nb"""\n')
         self.assertEqual(ip.user_ns['tmp'], 1)
 
     def test_dont_cache_with_semicolon(self):
         "Ending a line with semicolon should not cache the returned object (GH-307)"
         oldlen = len(ip.user_ns['Out'])
         a = ip.run_cell('1;', store_history=True)
         newlen = len(ip.user_ns['Out'])
         self.assertEqual(oldlen, newlen)
         #also test the default caching behavior
         ip.run_cell('1', store_history=True)
         newlen = len(ip.user_ns['Out'])
         self.assertEqual(oldlen+1, newlen)
 
     def test_In_variable(self):
         "Verify that In variable grows with user input (GH-284)"
         oldlen = len(ip.user_ns['In'])
         ip.run_cell('1;', store_history=True)
         newlen = len(ip.user_ns['In'])
         self.assertEqual(oldlen+1, newlen)
         self.assertEqual(ip.user_ns['In'][-1],'1;')
         
     def test_magic_names_in_string(self):
         ip.run_cell('a = """\n%exit\n"""')
         self.assertEqual(ip.user_ns['a'], '\n%exit\n')
     
     def test_trailing_newline(self):
         """test that running !(command) does not raise a SyntaxError"""
         ip.run_cell('!(true)\n', False)
         ip.run_cell('!(true)\n\n\n', False)
     
     def test_gh_597(self):
         """Pretty-printing lists of objects with non-ascii reprs may cause
         problems."""
         class Spam(object):
           def __repr__(self):
             return "\xe9"*50
         import IPython.core.formatters
         f = IPython.core.formatters.PlainTextFormatter()
         f([Spam(),Spam()])
     
 
     def test_future_flags(self):
         """Check that future flags are used for parsing code (gh-777)"""
         ip.run_cell('from __future__ import print_function')
         try:
             ip.run_cell('prfunc_return_val = print(1,2, sep=" ")')
             assert 'prfunc_return_val' in ip.user_ns
         finally:
             # Reset compiler flags so we don't mess up other tests.
             ip.compile.reset_compiler_flags()
 
     def test_future_unicode(self):
         """Check that unicode_literals is imported from __future__ (gh #786)"""
         try:
             ip.run_cell(u'byte_str = "a"')
             assert isinstance(ip.user_ns['byte_str'], str) # string literals are byte strings by default
             ip.run_cell('from __future__ import unicode_literals')
             ip.run_cell(u'unicode_str = "a"')
             assert isinstance(ip.user_ns['unicode_str'], unicode_type) # strings literals are now unicode
         finally:
             # Reset compiler flags so we don't mess up other tests.
             ip.compile.reset_compiler_flags()
     
     def test_can_pickle(self):
         "Can we pickle objects defined interactively (GH-29)"
         ip = get_ipython()
         ip.reset()
         ip.run_cell(("class Mylist(list):\n"
                      "    def __init__(self,x=[]):\n"
                      "        list.__init__(self,x)"))
         ip.run_cell("w=Mylist([1,2,3])")
         
         from pickle import dumps
         
         # We need to swap in our main module - this is only necessary
         # inside the test framework, because IPython puts the interactive module
         # in place (but the test framework undoes this).
         _main = sys.modules['__main__']
         sys.modules['__main__'] = ip.user_module
         try:
             res = dumps(ip.user_ns["w"])
         finally:
             sys.modules['__main__'] = _main
         self.assertTrue(isinstance(res, bytes))
         
     def test_global_ns(self):
         "Code in functions must be able to access variables outside them."
         ip = get_ipython()
         ip.run_cell("a = 10")
         ip.run_cell(("def f(x):\n"
                      "    return x + a"))
         ip.run_cell("b = f(12)")
         self.assertEqual(ip.user_ns["b"], 22)
 
     def test_bad_custom_tb(self):
         """Check that InteractiveShell is protected from bad custom exception handlers"""
         from IPython.utils import io
         save_stderr = io.stderr
         try:
             # capture stderr
             io.stderr = StringIO()
             ip.set_custom_exc((IOError,), lambda etype,value,tb: 1/0)
             self.assertEqual(ip.custom_exceptions, (IOError,))
             ip.run_cell(u'raise IOError("foo")')
             self.assertEqual(ip.custom_exceptions, ())
             self.assertTrue("Custom TB Handler failed" in io.stderr.getvalue())
         finally:
             io.stderr = save_stderr
 
     def test_bad_custom_tb_return(self):
         """Check that InteractiveShell is protected from bad return types in custom exception handlers"""
         from IPython.utils import io
         save_stderr = io.stderr
         try:
             # capture stderr
             io.stderr = StringIO()
             ip.set_custom_exc((NameError,),lambda etype,value,tb, tb_offset=None: 1)
             self.assertEqual(ip.custom_exceptions, (NameError,))
             ip.run_cell(u'a=abracadabra')
             self.assertEqual(ip.custom_exceptions, ())
             self.assertTrue("Custom TB Handler failed" in io.stderr.getvalue())
         finally:
             io.stderr = save_stderr
 
     def test_drop_by_id(self):
         myvars = {"a":object(), "b":object(), "c": object()}
         ip.push(myvars, interactive=False)
         for name in myvars:
             assert name in ip.user_ns, name
             assert name in ip.user_ns_hidden, name
         ip.user_ns['b'] = 12
         ip.drop_by_id(myvars)
         for name in ["a", "c"]:
             assert name not in ip.user_ns, name
             assert name not in ip.user_ns_hidden, name
         assert ip.user_ns['b'] == 12
         ip.reset()
 
     def test_var_expand(self):
         ip.user_ns['f'] = u'Ca\xf1o'
         self.assertEqual(ip.var_expand(u'echo $f'), u'echo Ca\xf1o')
         self.assertEqual(ip.var_expand(u'echo {f}'), u'echo Ca\xf1o')
         self.assertEqual(ip.var_expand(u'echo {f[:-1]}'), u'echo Ca\xf1')
         self.assertEqual(ip.var_expand(u'echo {1*2}'), u'echo 2')
 
         ip.user_ns['f'] = b'Ca\xc3\xb1o'
         # This should not raise any exception:
         ip.var_expand(u'echo $f')
     
     def test_var_expand_local(self):
         """Test local variable expansion in !system and %magic calls"""
         # !system
         ip.run_cell('def test():\n'
                     '    lvar = "ttt"\n'
                     '    ret = !echo {lvar}\n'
                     '    return ret[0]\n')
         res = ip.user_ns['test']()
         nt.assert_in('ttt', res)
         
         # %magic
         ip.run_cell('def makemacro():\n'
                     '    macroname = "macro_var_expand_locals"\n'
                     '    %macro {macroname} codestr\n')
         ip.user_ns['codestr'] = "str(12)"
         ip.run_cell('makemacro()')
         nt.assert_in('macro_var_expand_locals', ip.user_ns)
     
     def test_var_expand_self(self):
         """Test variable expansion with the name 'self', which was failing.
         
         See https://github.com/ipython/ipython/issues/1878#issuecomment-7698218
         """
         ip.run_cell('class cTest:\n'
                     '  classvar="see me"\n'
                     '  def test(self):\n'
                     '    res = !echo Variable: {self.classvar}\n'
                     '    return res[0]\n')
         nt.assert_in('see me', ip.user_ns['cTest']().test())
     
     def test_bad_var_expand(self):
         """var_expand on invalid formats shouldn't raise"""
         # SyntaxError
         self.assertEqual(ip.var_expand(u"{'a':5}"), u"{'a':5}")
         # NameError
         self.assertEqual(ip.var_expand(u"{asdf}"), u"{asdf}")
         # ZeroDivisionError
         self.assertEqual(ip.var_expand(u"{1/0}"), u"{1/0}")
     
     def test_silent_nopostexec(self):
         """run_cell(silent=True) doesn't invoke post-exec funcs"""
         d = dict(called=False)
         def set_called():
             d['called'] = True
         
         ip.register_post_execute(set_called)
         ip.run_cell("1", silent=True)
         self.assertFalse(d['called'])
         # double-check that non-silent exec did what we expected
         # silent to avoid
         ip.run_cell("1")
         self.assertTrue(d['called'])
         # remove post-exec
         ip._post_execute.pop(set_called)
     
     def test_silent_noadvance(self):
         """run_cell(silent=True) doesn't advance execution_count"""
         ec = ip.execution_count
         # silent should force store_history=False
         ip.run_cell("1", store_history=True, silent=True)
         
         self.assertEqual(ec, ip.execution_count)
         # double-check that non-silent exec did what we expected
         # silent to avoid
         ip.run_cell("1", store_history=True)
         self.assertEqual(ec+1, ip.execution_count)
     
     def test_silent_nodisplayhook(self):
         """run_cell(silent=True) doesn't trigger displayhook"""
         d = dict(called=False)
         
         trap = ip.display_trap
         save_hook = trap.hook
         
         def failing_hook(*args, **kwargs):
             d['called'] = True
         
         try:
             trap.hook = failing_hook
             ip.run_cell("1", silent=True)
             self.assertFalse(d['called'])
             # double-check that non-silent exec did what we expected
             # silent to avoid
             ip.run_cell("1")
             self.assertTrue(d['called'])
         finally:
             trap.hook = save_hook
 
     @skipif(sys.version_info[0] >= 3, "softspace removed in py3")
     def test_print_softspace(self):
         """Verify that softspace is handled correctly when executing multiple
         statements.
 
         In [1]: print 1; print 2
         1
         2
 
         In [2]: print 1,; print 2
         1 2
         """
         
     def test_ofind_line_magic(self):
         from IPython.core.magic import register_line_magic
         
         @register_line_magic
         def lmagic(line):
             "A line magic"
 
         # Get info on line magic
         lfind = ip._ofind('lmagic')
         info = dict(found=True, isalias=False, ismagic=True,
                     namespace = 'IPython internal', obj= lmagic.__wrapped__,
                     parent = None)
         nt.assert_equal(lfind, info)
         
     def test_ofind_cell_magic(self):
         from IPython.core.magic import register_cell_magic
         
         @register_cell_magic
         def cmagic(line, cell):
             "A cell magic"
 
         # Get info on cell magic
         find = ip._ofind('cmagic')
         info = dict(found=True, isalias=False, ismagic=True,
                     namespace = 'IPython internal', obj= cmagic.__wrapped__,
                     parent = None)
         nt.assert_equal(find, info)
     
     def test_custom_exception(self):
         called = []
         def my_handler(shell, etype, value, tb, tb_offset=None):
             called.append(etype)
             shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
         
         ip.set_custom_exc((ValueError,), my_handler)
         try:
             ip.run_cell("raise ValueError('test')")
             # Check that this was called, and only once.
             self.assertEqual(called, [ValueError])
         finally:
             # Reset the custom exception hook
             ip.set_custom_exc((), None)
     
     @skipif(sys.version_info[0] >= 3, "no differences with __future__ in py3")
     def test_future_environment(self):
         "Can we run code with & without the shell's __future__ imports?"
         ip.run_cell("from __future__ import division")
         ip.run_cell("a = 1/2", shell_futures=True)
         self.assertEqual(ip.user_ns['a'], 0.5)
         ip.run_cell("b = 1/2", shell_futures=False)
         self.assertEqual(ip.user_ns['b'], 0)
         
         ip.compile.reset_compiler_flags()
         # This shouldn't leak to the shell's compiler
         ip.run_cell("from __future__ import division \nc=1/2", shell_futures=False)
         self.assertEqual(ip.user_ns['c'], 0.5)
         ip.run_cell("d = 1/2", shell_futures=True)
         self.assertEqual(ip.user_ns['d'], 0)
 
 
 class TestSafeExecfileNonAsciiPath(unittest.TestCase):
 
     @onlyif_unicode_paths
     def setUp(self):
         self.BASETESTDIR = tempfile.mkdtemp()
         self.TESTDIR = join(self.BASETESTDIR, u"åäö")
         os.mkdir(self.TESTDIR)
         with open(join(self.TESTDIR, u"åäötestscript.py"), "w") as sfile:
             sfile.write("pass\n")
-        self.oldpath = os.getcwdu()
+        self.oldpath = py3compat.getcwd()
         os.chdir(self.TESTDIR)
         self.fname = u"åäötestscript.py"
 
     def tearDown(self):
         os.chdir(self.oldpath)
         shutil.rmtree(self.BASETESTDIR)
 
     @onlyif_unicode_paths
     def test_1(self):
         """Test safe_execfile with non-ascii path
         """
         ip.safe_execfile(self.fname, {}, raise_exceptions=True)
 
 class ExitCodeChecks(tt.TempFileMixin):
     def test_exit_code_ok(self):
         self.system('exit 0')
         self.assertEqual(ip.user_ns['_exit_code'], 0)
 
     def test_exit_code_error(self):
         self.system('exit 1')
         self.assertEqual(ip.user_ns['_exit_code'], 1)
 
     @skipif(not hasattr(signal, 'SIGALRM'))
     def test_exit_code_signal(self):
         self.mktmp("import signal, time\n"
                    "signal.setitimer(signal.ITIMER_REAL, 0.1)\n"
                    "time.sleep(1)\n")
         self.system("%s %s" % (sys.executable, self.fname))
         self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGALRM)
 
 class TestSystemRaw(unittest.TestCase, ExitCodeChecks):
     system = ip.system_raw
 
     @onlyif_unicode_paths
     def test_1(self):
         """Test system_raw with non-ascii cmd
         """
         cmd = u'''python -c "'åäö'"   '''
         ip.system_raw(cmd)
 
 # TODO: Exit codes are currently ignored on Windows.
 class TestSystemPipedExitCode(unittest.TestCase, ExitCodeChecks):
     system = ip.system_piped
 
     @skip_win32
     def test_exit_code_ok(self):
         ExitCodeChecks.test_exit_code_ok(self)
 
     @skip_win32
     def test_exit_code_error(self):
         ExitCodeChecks.test_exit_code_error(self)
 
     @skip_win32
     def test_exit_code_signal(self):
         ExitCodeChecks.test_exit_code_signal(self)
 
 class TestModules(unittest.TestCase, tt.TempFileMixin):
     def test_extraneous_loads(self):
         """Test we're not loading modules on startup that we shouldn't.
         """
         self.mktmp("import sys\n"
                    "print('numpy' in sys.modules)\n"
                    "print('IPython.parallel' in sys.modules)\n"
                    "print('IPython.kernel.zmq' in sys.modules)\n"
                    )
         out = "False\nFalse\nFalse\n"
         tt.ipexec_validate(self.fname, out)
 
 class Negator(ast.NodeTransformer):
     """Negates all number literals in an AST."""
     def visit_Num(self, node):
         node.n = -node.n
         return node
 
 class TestAstTransform(unittest.TestCase):
     def setUp(self):
         self.negator = Negator()
         ip.ast_transformers.append(self.negator)
     
     def tearDown(self):
         ip.ast_transformers.remove(self.negator)
     
     def test_run_cell(self):
         with tt.AssertPrints('-34'):
             ip.run_cell('print (12 + 22)')
         
         # A named reference to a number shouldn't be transformed.
         ip.user_ns['n'] = 55
         with tt.AssertNotPrints('-55'):
             ip.run_cell('print (n)')
     
     def test_timeit(self):
         called = set()
         def f(x):
             called.add(x)
         ip.push({'f':f})
         
         with tt.AssertPrints("best of "):
             ip.run_line_magic("timeit", "-n1 f(1)")
         self.assertEqual(called, set([-1]))
         called.clear()
         
         with tt.AssertPrints("best of "):
             ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
         self.assertEqual(called, set([-2, -3]))
     
     def test_time(self):
         called = []
         def f(x):
             called.append(x)
         ip.push({'f':f})
         
         # Test with an expression
         with tt.AssertPrints("Wall time: "):
             ip.run_line_magic("time", "f(5+9)")
         self.assertEqual(called, [-14])
         called[:] = []
         
         # Test with a statement (different code path)
         with tt.AssertPrints("Wall time: "):
             ip.run_line_magic("time", "a = f(-3 + -2)")
         self.assertEqual(called, [5])
     
     def test_macro(self):
         ip.push({'a':10})
         # The AST transformation makes this do a+=-1
         ip.define_macro("amacro", "a+=1\nprint(a)")
         
         with tt.AssertPrints("9"):
             ip.run_cell("amacro")
         with tt.AssertPrints("8"):
             ip.run_cell("amacro")
 
 class IntegerWrapper(ast.NodeTransformer):
     """Wraps all integers in a call to Integer()"""
     def visit_Num(self, node):
         if isinstance(node.n, int):
             return ast.Call(func=ast.Name(id='Integer', ctx=ast.Load()),
                             args=[node], keywords=[])
         return node
 
 class TestAstTransform2(unittest.TestCase):
     def setUp(self):
         self.intwrapper = IntegerWrapper()
         ip.ast_transformers.append(self.intwrapper)
         
         self.calls = []
         def Integer(*args):
             self.calls.append(args)
             return args
         ip.push({"Integer": Integer})
     
     def tearDown(self):
         ip.ast_transformers.remove(self.intwrapper)
         del ip.user_ns['Integer']
     
     def test_run_cell(self):
         ip.run_cell("n = 2")
         self.assertEqual(self.calls, [(2,)])
         
         # This shouldn't throw an error
         ip.run_cell("o = 2.0")
         self.assertEqual(ip.user_ns['o'], 2.0)
     
     def test_timeit(self):
         called = set()
         def f(x):
             called.add(x)
         ip.push({'f':f})
         
         with tt.AssertPrints("best of "):
             ip.run_line_magic("timeit", "-n1 f(1)")
         self.assertEqual(called, set([(1,)]))
         called.clear()
         
         with tt.AssertPrints("best of "):
             ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
         self.assertEqual(called, set([(2,), (3,)]))
 
 class ErrorTransformer(ast.NodeTransformer):
     """Throws an error when it sees a number."""
     def visit_Num(self):
         raise ValueError("test")
 
 class TestAstTransformError(unittest.TestCase):
     def test_unregistering(self):
         err_transformer = ErrorTransformer()
         ip.ast_transformers.append(err_transformer)
         
         with tt.AssertPrints("unregister", channel='stderr'):
             ip.run_cell("1 + 2")
         
         # This should have been removed.
         nt.assert_not_in(err_transformer, ip.ast_transformers)
 
 def test__IPYTHON__():
     # This shouldn't raise a NameError, that's all
     __IPYTHON__
 
 
 class DummyRepr(object):
     def __repr__(self):
         return "DummyRepr"
     
     def _repr_html_(self):
         return "<b>dummy</b>"
     
     def _repr_javascript_(self):
         return "console.log('hi');", {'key': 'value'}
     
 
 def test_user_variables():
     # enable all formatters
     ip.display_formatter.active_types = ip.display_formatter.format_types
     
     ip.user_ns['dummy'] = d = DummyRepr()
     keys = set(['dummy', 'doesnotexist'])
     r = ip.user_variables(keys)
 
     nt.assert_equal(keys, set(r.keys()))
     dummy = r['dummy']
     nt.assert_equal(set(['status', 'data', 'metadata']), set(dummy.keys()))
     nt.assert_equal(dummy['status'], 'ok')
     data = dummy['data']
     metadata = dummy['metadata']
     nt.assert_equal(data.get('text/html'), d._repr_html_())
     js, jsmd = d._repr_javascript_()
     nt.assert_equal(data.get('application/javascript'), js)
     nt.assert_equal(metadata.get('application/javascript'), jsmd)
     
     dne = r['doesnotexist']
     nt.assert_equal(dne['status'], 'error')
     nt.assert_equal(dne['ename'], 'KeyError')
     
     # back to text only
     ip.display_formatter.active_types = ['text/plain']
     
 def test_user_expression():
     # enable all formatters
     ip.display_formatter.active_types = ip.display_formatter.format_types
     query = {
         'a' : '1 + 2',
         'b' : '1/0',
     }
     r = ip.user_expressions(query)
     import pprint
     pprint.pprint(r)
     nt.assert_equal(r.keys(), query.keys())
     a = r['a']
     nt.assert_equal(set(['status', 'data', 'metadata']), set(a.keys()))
     nt.assert_equal(a['status'], 'ok')
     data = a['data']
     metadata = a['metadata']
     nt.assert_equal(data.get('text/plain'), '3')
     
     b = r['b']
     nt.assert_equal(b['status'], 'error')
     nt.assert_equal(b['ename'], 'ZeroDivisionError')
     
     # back to text only
     ip.display_formatter.active_types = ['text/plain']
     
 
 
 
 
 

IPython/core/tests/test_magic.py

@@ -1,944 +1,944 b''
 # -*- coding: utf-8 -*-
 """Tests for various magic functions.
 
 Needs to be run by nose (to make ipython session available).
 """
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import io
 import os
 import sys
 from unittest import TestCase
 
 try:
     from importlib import invalidate_caches   # Required from Python 3.3
 except ImportError:
     def invalidate_caches():
         pass
 
 import nose.tools as nt
 
 from IPython.core import magic
 from IPython.core.magic import (Magics, magics_class, line_magic,
                                 cell_magic, line_cell_magic,
                                 register_line_magic, register_cell_magic,
                                 register_line_cell_magic)
 from IPython.core.magics import execution, script, code
 from IPython.nbformat.v3.tests.nbexamples import nb0
 from IPython.nbformat import current
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils import py3compat
 from IPython.utils.io import capture_output
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.utils.process import find_cmd
 
 if py3compat.PY3:
     from io import StringIO
 else:
     from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 
 @magic.magics_class
 class DummyMagics(magic.Magics): pass
 
 def test_extract_code_ranges():
     instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
     expected = [(0, 1), 
                 (2, 3),
                 (4, 6),
                 (6, 9),
                 (9, 14),
                 (16, None),
                 (None, 9),
                 (9, None),
                 (None, 13),
                 (None, None)]
     actual = list(code.extract_code_ranges(instr))
     nt.assert_equal(actual, expected)
 
 def test_extract_symbols():
     source = """import foo\na = 10\ndef b():\n    return 42\n\n\nclass A: pass\n\n\n"""
     symbols_args = ["a", "b", "A", "A,b", "A,a", "z"]
     expected = [([], ['a']),
                 (["def b():\n    return 42\n"], []),
                 (["class A: pass\n"], []),
                 (["class A: pass\n", "def b():\n    return 42\n"], []),
                 (["class A: pass\n"], ['a']),
                 ([], ['z'])]
     for symbols, exp in zip(symbols_args, expected):
         nt.assert_equal(code.extract_symbols(source, symbols), exp)
 
 
 def test_extract_symbols_raises_exception_with_non_python_code():
     source = ("=begin A Ruby program :)=end\n"
               "def hello\n"
               "puts 'Hello world'\n"
               "end")
     with nt.assert_raises(SyntaxError):
         code.extract_symbols(source, "hello")
 
 def test_config():
     """ test that config magic does not raise
     can happen if Configurable init is moved too early into
     Magics.__init__ as then a Config object will be registerd as a
     magic.
     """
     ## should not raise.
     _ip.magic('config')
 
 def test_rehashx():
     # clear up everything
     _ip = get_ipython()
     _ip.alias_manager.clear_aliases()
     del _ip.db['syscmdlist']
     
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
     nt.assert_true(len(_ip.alias_manager.aliases) > 10)
     for name, cmd in _ip.alias_manager.aliases:
         # we must strip dots from alias names
         nt.assert_not_in('.', name)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
     nt.assert_true(len(scoms) > 10)
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
     path = 'c:\\x'
     m = DummyMagics(ip)
     opts = m.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
     if os.name == 'posix':
         expected = 'c:x'
     else:
         expected = path
     nt.assert_equal(opts['f'], expected)
 
 def test_magic_parse_long_options():
     """Magic.parse_options can handle --foo=bar long options"""
     ip = get_ipython()
     m = DummyMagics(ip)
     opts, _ = m.parse_options('--foo --bar=bubble', 'a', 'foo', 'bar=')
     nt.assert_in('foo', opts)
     nt.assert_in('bar', opts)
     nt.assert_equal(opts['bar'], "bubble")
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -nl -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_r():
     """Test %hist -r
 
     XXX - This test is not recording the output correctly.  For some reason, in
     testing mode the raw history isn't getting populated.  No idea why.
     Disabling the output checking for now, though at least we do run it.
 
     In [1]: 'hist' in _ip.lsmagic()
     Out[1]: True
 
     In [2]: x=1
 
     In [3]: %hist -rl 2
     x=1 # random
     %hist -r 2
     """
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b(float):
        ...:     pass
        ...: 
 
     In [2]: class s(object):
        ...:     def __str__(self):
        ...:         return 's'
        ...: 
 
     In [3]: 
 
     In [4]: class r(b):
        ...:     def __repr__(self):
        ...:         return 'r'
        ...: 
 
     In [5]: class sr(s,r): pass
        ...: 
 
     In [6]: 
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: 4.5
     Out[11]: 4.5
 
     In [12]: str(ss)
     Out[12]: 's'
 
     In [13]: 
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ... 
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ... 
     >>> 
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ... 
     >>> class sr(s,r): pass
     >>> 
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> 4.5
     4.5
     >>> str(ss)
     's'
     >>> 
     """
 
 
 @dec.skip_without('sqlite3')
 def test_macro():
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print(a,b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     ip.magic("macro test 1-3")
     nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
     
     # List macros
     nt.assert_in("test", ip.magic("macro"))
 
 
 @dec.skip_without('sqlite3')
 def test_macro_run():
     """Test that we can run a multi-line macro successfully."""
     ip = get_ipython()
     ip.history_manager.reset()
     cmds = ["a=10", "a+=1", py3compat.doctest_refactor_print("print a"),
                                                             "%macro test 2-3"]
     for cmd in cmds:
         ip.run_cell(cmd, store_history=True)
     nt.assert_equal(ip.user_ns["test"].value,
                             py3compat.doctest_refactor_print("a+=1\nprint a\n"))
     with tt.AssertPrints("12"):
         ip.run_cell("test")
     with tt.AssertPrints("13"):
         ip.run_cell("test")
 
 
 def test_magic_magic():
     """Test %magic"""
     ip = get_ipython()
     with capture_output() as captured:
         ip.magic("magic")
     
     stdout = captured.stdout
     nt.assert_in('%magic', stdout)
     nt.assert_in('IPython', stdout)
     nt.assert_in('Available', stdout)
 
 
 @dec.skipif_not_numpy
 def test_numpy_reset_array_undec():
     "Test '%reset array' functionality"
     _ip.ex('import numpy as np')
     _ip.ex('a = np.empty(2)')
     nt.assert_in('a', _ip.user_ns)
     _ip.magic('reset -f array')
     nt.assert_not_in('a', _ip.user_ns)
 
 def test_reset_out():
     "Test '%reset out' magic"
     _ip.run_cell("parrot = 'dead'", store_history=True)
     # test '%reset -f out', make an Out prompt
     _ip.run_cell("parrot", store_history=True)
     nt.assert_true('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
     _ip.magic('reset -f out')
     nt.assert_false('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
     nt.assert_equal(len(_ip.user_ns['Out']), 0)
 
 def test_reset_in():
     "Test '%reset in' magic"
     # test '%reset -f in'
     _ip.run_cell("parrot", store_history=True)
     nt.assert_true('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
     _ip.magic('%reset -f in')
     nt.assert_false('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
     nt.assert_equal(len(set(_ip.user_ns['In'])), 1)
 
 def test_reset_dhist():
     "Test '%reset dhist' magic"
     _ip.run_cell("tmp = [d for d in _dh]") # copy before clearing
     _ip.magic('cd ' + os.path.dirname(nt.__file__))
     _ip.magic('cd -')
     nt.assert_true(len(_ip.user_ns['_dh']) > 0)
     _ip.magic('reset -f dhist')
     nt.assert_equal(len(_ip.user_ns['_dh']), 0)
     _ip.run_cell("_dh = [d for d in tmp]") #restore
 
 def test_reset_in_length():
     "Test that '%reset in' preserves In[] length"
     _ip.run_cell("print 'foo'")
     _ip.run_cell("reset -f in")
     nt.assert_equal(len(_ip.user_ns['In']), _ip.displayhook.prompt_count+1)
 
 def test_tb_syntaxerror():
     """test %tb after a SyntaxError"""
     ip = get_ipython()
     ip.run_cell("for")
     
     # trap and validate stdout
     save_stdout = sys.stdout
     try:
         sys.stdout = StringIO()
         ip.run_cell("%tb")
         out = sys.stdout.getvalue()
     finally:
         sys.stdout = save_stdout
     # trim output, and only check the last line
     last_line = out.rstrip().splitlines()[-1].strip()
     nt.assert_equal(last_line, "SyntaxError: invalid syntax")
 
 
 def test_time():
     ip = get_ipython()
     
     with tt.AssertPrints("Wall time: "):
         ip.run_cell("%time None")
     
     ip.run_cell("def f(kmjy):\n"
                 "    %time print (2*kmjy)")
     
     with tt.AssertPrints("Wall time: "):
         with tt.AssertPrints("hihi", suppress=False):
             ip.run_cell("f('hi')")
 
 
 @dec.skip_win32
 def test_time2():
     ip = get_ipython()
     
     with tt.AssertPrints("CPU times: user "):
         ip.run_cell("%time None")
 
 def test_time3():
     """Erroneous magic function calls, issue gh-3334"""
     ip = get_ipython()
     ip.user_ns.pop('run', None)
     
     with tt.AssertNotPrints("not found", channel='stderr'):
         ip.run_cell("%%time\n"
                     "run = 0\n"
                     "run += 1")
 
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.magic('doctest_mode')
     _ip.magic('doctest_mode')
 
 
 def test_parse_options():
     """Tests for basic options parsing in magics."""
     # These are only the most minimal of tests, more should be added later.  At
     # the very least we check that basic text/unicode calls work OK.
     m = DummyMagics(_ip)
     nt.assert_equal(m.parse_options('foo', '')[1], 'foo')
     nt.assert_equal(m.parse_options(u'foo', '')[1], u'foo')
 
     
 def test_dirops():
     """Test various directory handling operations."""
-    # curpath = lambda :os.path.splitdrive(os.getcwdu())[1].replace('\\','/')
-    curpath = os.getcwdu
-    startdir = os.getcwdu()
+    # curpath = lambda :os.path.splitdrive(py3compat.getcwd())[1].replace('\\','/')
+    curpath = py3compat.getcwd
+    startdir = py3compat.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
     try:
         _ip.magic('cd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('cd -')
         nt.assert_equal(curpath(), startdir)
         _ip.magic('pushd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('popd')
         nt.assert_equal(curpath(), startdir)
     finally:
         os.chdir(startdir)
 
 
 def test_xmode():
     # Calling xmode three times should be a no-op
     xmode = _ip.InteractiveTB.mode
     for i in range(3):
         _ip.magic("xmode")
     nt.assert_equal(_ip.InteractiveTB.mode, xmode)
     
 def test_reset_hard():
     monitor = []
     class A(object):
         def __del__(self):
             monitor.append(1)
         def __repr__(self):
             return "<A instance>"
             
     _ip.user_ns["a"] = A()
     _ip.run_cell("a")
     
     nt.assert_equal(monitor, [])
     _ip.magic("reset -f")
     nt.assert_equal(monitor, [1])
     
 class TestXdel(tt.TempFileMixin):
     def test_xdel(self):
         """Test that references from %run are cleared by xdel."""
         src = ("class A(object):\n"
                "    monitor = []\n"
                "    def __del__(self):\n"
                "        self.monitor.append(1)\n"
                "a = A()\n")
         self.mktmp(src)
         # %run creates some hidden references...
         _ip.magic("run %s" % self.fname)
         # ... as does the displayhook.
         _ip.run_cell("a")
         
         monitor = _ip.user_ns["A"].monitor
         nt.assert_equal(monitor, [])
         
         _ip.magic("xdel a")
         
         # Check that a's __del__ method has been called.
         nt.assert_equal(monitor, [1])
 
 def doctest_who():
     """doctest for %who
     
     In [1]: %reset -f
     
     In [2]: alpha = 123
     
     In [3]: beta = 'beta'
     
     In [4]: %who int
     alpha
     
     In [5]: %who str
     beta
     
     In [6]: %whos
     Variable   Type    Data/Info
     ----------------------------
     alpha      int     123
     beta       str     beta
     
     In [7]: %who_ls
     Out[7]: ['alpha', 'beta']
     """
 
 def test_whos():
     """Check that whos is protected against objects where repr() fails."""
     class A(object):
         def __repr__(self):
             raise Exception()
     _ip.user_ns['a'] = A()
     _ip.magic("whos")
 
 @py3compat.u_format
 def doctest_precision():
     """doctest for %precision
     
     In [1]: f = get_ipython().display_formatter.formatters['text/plain']
     
     In [2]: %precision 5
     Out[2]: {u}'%.5f'
     
     In [3]: f.float_format
     Out[3]: {u}'%.5f'
     
     In [4]: %precision %e
     Out[4]: {u}'%e'
     
     In [5]: f(3.1415927)
     Out[5]: {u}'3.141593e+00'
     """
 
 def test_psearch():
     with tt.AssertPrints("dict.fromkeys"):
         _ip.run_cell("dict.fr*?")
 
 def test_timeit_shlex():
     """test shlex issues with timeit (#1109)"""
     _ip.ex("def f(*a,**kw): pass")
     _ip.magic('timeit -n1 "this is a bug".count(" ")')
     _ip.magic('timeit -r1 -n1 f(" ", 1)')
     _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
     _ip.magic('timeit -r1 -n1 ("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b ")')
 
 
 def test_timeit_arguments():
     "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
     _ip.magic("timeit ('#')")
 
 
 def test_timeit_special_syntax():
     "Test %%timeit with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic('timeit', '-n1 -r1 %lmagic my line')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
     # cell mode test
     _ip.run_cell_magic('timeit', '-n1 -r1', '%lmagic my line2')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
 
 def test_timeit_return():
     """
     test wether timeit -o return object
     """
 
     res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
     assert(res is not None)
 
 def test_timeit_quiet():
     """
     test quiet option of timeit magic
     """
     with tt.AssertNotPrints("loops"):
         _ip.run_cell("%timeit -n1 -r1 -q 1")
 
 @dec.skipif(execution.profile is None)
 def test_prun_special_syntax():
     "Test %%prun with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic('prun', '-q %lmagic my line')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
     # cell mode test
     _ip.run_cell_magic('prun', '-q', '%lmagic my line2')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
 
 @dec.skipif(execution.profile is None)
 def test_prun_quotes():
     "Test that prun does not clobber string escapes (GH #1302)"
     _ip.magic(r"prun -q x = '\t'")
     nt.assert_equal(_ip.user_ns['x'], '\t')
 
 def test_extension():
     tmpdir = TemporaryDirectory()
     orig_ipython_dir = _ip.ipython_dir
     try:
         _ip.ipython_dir = tmpdir.name
         nt.assert_raises(ImportError, _ip.magic, "load_ext daft_extension")
         url = os.path.join(os.path.dirname(__file__), "daft_extension.py")
         _ip.magic("install_ext %s" % url)
         _ip.user_ns.pop('arq', None)
         invalidate_caches()   # Clear import caches
         _ip.magic("load_ext daft_extension")
         nt.assert_equal(_ip.user_ns['arq'], 185)
         _ip.magic("unload_ext daft_extension")
         assert 'arq' not in _ip.user_ns
     finally:
         _ip.ipython_dir = orig_ipython_dir
         tmpdir.cleanup()
         
 def test_notebook_export_json():
     with TemporaryDirectory() as td:
         outfile = os.path.join(td, "nb.ipynb")
         _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
         _ip.magic("notebook -e %s" % outfile)
 
 def test_notebook_export_py():
     with TemporaryDirectory() as td:
         outfile = os.path.join(td, "nb.py")
         _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
         _ip.magic("notebook -e %s" % outfile)
 
 def test_notebook_reformat_py():
     with TemporaryDirectory() as td:
         infile = os.path.join(td, "nb.ipynb")
         with io.open(infile, 'w', encoding='utf-8') as f:
             current.write(nb0, f, 'json')
             
         _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
         _ip.magic("notebook -f py %s" % infile)
 
 def test_notebook_reformat_json():
     with TemporaryDirectory() as td:
         infile = os.path.join(td, "nb.py")
         with io.open(infile, 'w', encoding='utf-8') as f:
             current.write(nb0, f, 'py')
             
         _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
         _ip.magic("notebook -f ipynb %s" % infile)
         _ip.magic("notebook -f json %s" % infile)
 
 def test_env():
     env = _ip.magic("env")
     assert isinstance(env, dict), type(env)
 
 
 class CellMagicTestCase(TestCase):
 
     def check_ident(self, magic):
         # Manually called, we get the result
         out = _ip.run_cell_magic(magic, 'a', 'b')
         nt.assert_equal(out, ('a','b'))
         # Via run_cell, it goes into the user's namespace via displayhook
         _ip.run_cell('%%' + magic +' c\nd')
         nt.assert_equal(_ip.user_ns['_'], ('c','d'))
 
     def test_cell_magic_func_deco(self):
         "Cell magic using simple decorator"
         @register_cell_magic
         def cellm(line, cell):
             return line, cell
 
         self.check_ident('cellm')
 
     def test_cell_magic_reg(self):
         "Cell magic manually registered"
         def cellm(line, cell):
             return line, cell
 
         _ip.register_magic_function(cellm, 'cell', 'cellm2')
         self.check_ident('cellm2')
 
     def test_cell_magic_class(self):
         "Cell magics declared via a class"
         @magics_class
         class MyMagics(Magics):
 
             @cell_magic
             def cellm3(self, line, cell):
                 return line, cell
 
         _ip.register_magics(MyMagics)
         self.check_ident('cellm3')
 
     def test_cell_magic_class2(self):
         "Cell magics declared via a class, #2"
         @magics_class
         class MyMagics2(Magics):
 
             @cell_magic('cellm4')
             def cellm33(self, line, cell):
                 return line, cell
             
         _ip.register_magics(MyMagics2)
         self.check_ident('cellm4')
         # Check that nothing is registered as 'cellm33'
         c33 = _ip.find_cell_magic('cellm33')
         nt.assert_equal(c33, None)
 
 def test_file():
     """Basic %%file"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.run_cell_magic("file", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
 def test_file_var_expand():
     """%%file $filename"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.user_ns['filename'] = fname
         ip.run_cell_magic("file", '$filename', u'\n'.join([
             'line1',
             'line2',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
 def test_file_unicode():
     """%%file with unicode cell"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.run_cell_magic("file", fname, u'\n'.join([
             u'liné1',
             u'liné2',
         ]))
         with io.open(fname, encoding='utf-8') as f:
             s = f.read()
         nt.assert_in(u'liné1\n', s)
         nt.assert_in(u'liné2', s)
 
 def test_file_amend():
     """%%file -a amends files"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file2')
         ip.run_cell_magic("file", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         ip.run_cell_magic("file", "-a %s" % fname, u'\n'.join([
             'line3',
             'line4',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line3\n', s)
         
     
 def test_script_config():
     ip = get_ipython()
     ip.config.ScriptMagics.script_magics = ['whoda']
     sm = script.ScriptMagics(shell=ip)
     nt.assert_in('whoda', sm.magics['cell'])
 
 @dec.skip_win32
 def test_script_out():
     ip = get_ipython()
     ip.run_cell_magic("script", "--out output sh", "echo 'hi'")
     nt.assert_equal(ip.user_ns['output'], 'hi\n')
 
 @dec.skip_win32
 def test_script_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--err error sh", "echo 'hello' >&2")
     nt.assert_equal(ip.user_ns['error'], 'hello\n')
 
 @dec.skip_win32
 def test_script_out_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2")
     nt.assert_equal(ip.user_ns['output'], 'hi\n')
     nt.assert_equal(ip.user_ns['error'], 'hello\n')
 
 @dec.skip_win32
 def test_script_bg_out():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'")
     nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
 
 @dec.skip_win32
 def test_script_bg_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --err error sh", "echo 'hello' >&2")
     nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
 
 @dec.skip_win32
 def test_script_bg_out_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --out output --err error sh", "echo 'hi'\necho 'hello' >&2")
     nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
     nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
 
 def test_script_defaults():
     ip = get_ipython()
     for cmd in ['sh', 'bash', 'perl', 'ruby']:
         try:
             find_cmd(cmd)
         except Exception:
             pass
         else:
             nt.assert_in(cmd, ip.magics_manager.magics['cell'])
 
 
 @magics_class
 class FooFoo(Magics):
     """class with both %foo and %%foo magics"""
     @line_magic('foo')
     def line_foo(self, line):
         "I am line foo"
         pass
 
     @cell_magic("foo")
     def cell_foo(self, line, cell):
         "I am cell foo, not line foo"
         pass
 
 def test_line_cell_info():
     """%%foo and %foo magics are distinguishable to inspect"""
     ip = get_ipython()
     ip.magics_manager.register(FooFoo)
     oinfo = ip.object_inspect('foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     
     oinfo = ip.object_inspect('%%foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     nt.assert_equal(oinfo['docstring'], FooFoo.cell_foo.__doc__)
 
     oinfo = ip.object_inspect('%foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     nt.assert_equal(oinfo['docstring'], FooFoo.line_foo.__doc__)
 
 def test_multiple_magics():
     ip = get_ipython()
     foo1 = FooFoo(ip)
     foo2 = FooFoo(ip)
     mm = ip.magics_manager
     mm.register(foo1)
     nt.assert_true(mm.magics['line']['foo'].__self__ is foo1)
     mm.register(foo2)
     nt.assert_true(mm.magics['line']['foo'].__self__ is foo2)
 
 def test_alias_magic():
     """Test %alias_magic."""
     ip = get_ipython()
     mm = ip.magics_manager
 
     # Basic operation: both cell and line magics are created, if possible.
     ip.run_line_magic('alias_magic', 'timeit_alias timeit')
     nt.assert_in('timeit_alias', mm.magics['line'])
     nt.assert_in('timeit_alias', mm.magics['cell'])
 
     # --cell is specified, line magic not created.
     ip.run_line_magic('alias_magic', '--cell timeit_cell_alias timeit')
     nt.assert_not_in('timeit_cell_alias', mm.magics['line'])
     nt.assert_in('timeit_cell_alias', mm.magics['cell'])
 
     # Test that line alias is created successfully.
     ip.run_line_magic('alias_magic', '--line env_alias env')
     nt.assert_equal(ip.run_line_magic('env', ''),
                     ip.run_line_magic('env_alias', ''))
 
 def test_save():
     """Test %save."""
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = [u"a=1", u"def b():\n  return a**2", u"print(a, b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as tmpdir:
         file = os.path.join(tmpdir, "testsave.py")
         ip.run_line_magic("save", "%s 1-10" % file)
         with open(file) as f:
             content = f.read()
             nt.assert_equal(content.count(cmds[0]), 1)
             nt.assert_in('coding: utf-8', content)
         ip.run_line_magic("save", "-a %s 1-10" % file)
         with open(file) as f:
             content = f.read()
             nt.assert_equal(content.count(cmds[0]), 2)
             nt.assert_in('coding: utf-8', content)
 
 
 def test_store():
     """Test %store."""
     ip = get_ipython()
     ip.run_line_magic('load_ext', 'storemagic')
     
     # make sure the storage is empty
     ip.run_line_magic('store', '-z')
     ip.user_ns['var'] = 42
     ip.run_line_magic('store', 'var')
     ip.user_ns['var'] = 39
     ip.run_line_magic('store', '-r')
     nt.assert_equal(ip.user_ns['var'], 42)
 
     ip.run_line_magic('store', '-d var')
     ip.user_ns['var'] = 39
     ip.run_line_magic('store' , '-r')
     nt.assert_equal(ip.user_ns['var'], 39)
 
 
 def _run_edit_test(arg_s, exp_filename=None,
                         exp_lineno=-1,
                         exp_contents=None,
                         exp_is_temp=None):
     ip = get_ipython()
     M = code.CodeMagics(ip)
     last_call = ['','']
     opts,args = M.parse_options(arg_s,'prxn:')
     filename, lineno, is_temp = M._find_edit_target(ip, args, opts, last_call)
         
     if exp_filename is not None:
         nt.assert_equal(exp_filename, filename)
     if exp_contents is not None:
         with io.open(filename, 'r') as f:
             contents = f.read()
         nt.assert_equal(exp_contents, contents)
     if exp_lineno != -1:
         nt.assert_equal(exp_lineno, lineno)
     if exp_is_temp is not None:
         nt.assert_equal(exp_is_temp, is_temp)
 
 
 def test_edit_interactive():
     """%edit on interactively defined objects"""
     ip = get_ipython()
     n = ip.execution_count
     ip.run_cell(u"def foo(): return 1", store_history=True)
     
     try:
         _run_edit_test("foo")
     except code.InteractivelyDefined as e:
         nt.assert_equal(e.index, n)
     else:
         raise AssertionError("Should have raised InteractivelyDefined")
 
 
 def test_edit_cell():
     """%edit [cell id]"""
     ip = get_ipython()
     
     ip.run_cell(u"def foo(): return 1", store_history=True)
     
     # test
     _run_edit_test("1", exp_contents=ip.user_ns['In'][1], exp_is_temp=True)

IPython/core/tests/test_prompts.py

@@ -1,111 +1,112 b''
 # -*- coding: utf-8
 """Tests for prompt generation."""
 
 import unittest
 
 import os
 
 from IPython.testing import tools as tt, decorators as dec
 from IPython.core.prompts import PromptManager, LazyEvaluate
 from IPython.testing.globalipapp import get_ipython
 from IPython.utils.tempdir import TemporaryDirectory
+from IPython.utils import py3compat
 from IPython.utils.py3compat import unicode_type
 
 ip = get_ipython()
 
 
 class PromptTests(unittest.TestCase):
     def setUp(self):
         self.pm = PromptManager(shell=ip, config=ip.config)
     
     def test_multiline_prompt(self):
         self.pm.in_template = "[In]\n>>>"
         self.pm.render('in')
         self.assertEqual(self.pm.width, 3)
         self.assertEqual(self.pm.txtwidth, 3)
         
         self.pm.in_template = '[In]\n'
         self.pm.render('in')
         self.assertEqual(self.pm.width, 0)
         self.assertEqual(self.pm.txtwidth, 0)
     
     def test_translate_abbreviations(self):
         def do_translate(template):
             self.pm.in_template = template
             return self.pm.templates['in']
         
         pairs = [(r'%n>', '{color.number}{count}{color.prompt}>'),
                  (r'\T', '{time}'),
                  (r'\n', '\n')
                 ]
     
         tt.check_pairs(do_translate, pairs)
     
     def test_user_ns(self):
         self.pm.color_scheme = 'NoColor'
         ip.ex("foo='bar'")
         self.pm.in_template = "In [{foo}]"
         prompt = self.pm.render('in')
         self.assertEqual(prompt, u'In [bar]')
 
     def test_builtins(self):
         self.pm.color_scheme = 'NoColor'
         self.pm.in_template = "In [{int}]"
         prompt = self.pm.render('in')
         self.assertEqual(prompt, u"In [%r]" % int)
 
     def test_undefined(self):
         self.pm.color_scheme = 'NoColor'
         self.pm.in_template = "In [{foo_dne}]"
         prompt = self.pm.render('in')
         self.assertEqual(prompt, u"In [<ERROR: 'foo_dne' not found>]")
 
     def test_render(self):
         self.pm.in_template = r'\#>'
         self.assertEqual(self.pm.render('in',color=False), '%d>' % ip.execution_count)
     
     @dec.onlyif_unicode_paths
     def test_render_unicode_cwd(self):
-        save = os.getcwdu()
+        save = py3compat.getcwd()
         with TemporaryDirectory(u'ünicødé') as td:
             os.chdir(td)
             self.pm.in_template = r'\w [\#]'
             p = self.pm.render('in', color=False)
-            self.assertEqual(p, u"%s [%i]" % (os.getcwdu(), ip.execution_count))
+            self.assertEqual(p, u"%s [%i]" % (py3compat.getcwd(), ip.execution_count))
         os.chdir(save)
     
     def test_lazy_eval_unicode(self):
         u = u'ünicødé'
         lz = LazyEvaluate(lambda : u)
         # str(lz) would fail
         self.assertEqual(unicode_type(lz), u)
         self.assertEqual(format(lz), u)
     
     def test_lazy_eval_nonascii_bytes(self):
         u = u'ünicødé'
         b = u.encode('utf8')
         lz = LazyEvaluate(lambda : b)
         # unicode(lz) would fail
         self.assertEqual(str(lz), str(b))
         self.assertEqual(format(lz), str(b))
     
     def test_lazy_eval_float(self):
         f = 0.503
         lz = LazyEvaluate(lambda : f)
         
         self.assertEqual(str(lz), str(f))
         self.assertEqual(unicode_type(lz), unicode_type(f))
         self.assertEqual(format(lz), str(f))
         self.assertEqual(format(lz, '.1'), '0.5')
     
     @dec.skip_win32
     def test_cwd_x(self):
         self.pm.in_template = r"\X0"
-        save = os.getcwdu()
+        save = py3compat.getcwd()
         os.chdir(os.path.expanduser('~'))
         p = self.pm.render('in', color=False)
         try:
             self.assertEqual(p, '~')
         finally:
             os.chdir(save)
     

IPython/core/tests/test_run.py

@@ -1,458 +1,458 b''
 # encoding: utf-8
 """Tests for code execution (%run and related), which is particularly tricky.
 
 Because of how %run manages namespaces, and the fact that we are trying here to
 verify subtle object deletion and reference counting issues, the %run tests
 will be kept in this separate file.  This makes it easier to aggregate in one
 place the tricks needed to handle it; most other magics are much easier to test
 and we do so in a common test_magic file.
 """
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import functools
 import os
 from os.path import join as pjoin
 import random
 import sys
 import tempfile
 import textwrap
 import unittest
 
 import nose.tools as nt
 from nose import SkipTest
 
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils import py3compat
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.core import debugger
 
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 
 def doctest_refbug():
     """Very nasty problem with references held by multiple runs of a script.
     See: https://github.com/ipython/ipython/issues/141
 
     In [1]: _ip.clear_main_mod_cache()
     # random
 
     In [2]: %run refbug
 
     In [3]: call_f()
     lowercased: hello
 
     In [4]: %run refbug
 
     In [5]: call_f()
     lowercased: hello
     lowercased: hello
     """
 
 
 def doctest_run_builtins():
     r"""Check that %run doesn't damage __builtins__.
 
     In [1]: import tempfile
 
     In [2]: bid1 = id(__builtins__)
 
     In [3]: fname = tempfile.mkstemp('.py')[1]
 
     In [3]: f = open(fname,'w')
 
     In [4]: dummy= f.write('pass\n')
 
     In [5]: f.flush()
 
     In [6]: t1 = type(__builtins__)
 
     In [7]: %run $fname
 
     In [7]: f.close()
 
     In [8]: bid2 = id(__builtins__)
 
     In [9]: t2 = type(__builtins__)
 
     In [10]: t1 == t2
     Out[10]: True
 
     In [10]: bid1 == bid2
     Out[10]: True
 
     In [12]: try:
        ....:     os.unlink(fname)
        ....: except:
        ....:     pass
        ....:
     """
 
 
 def doctest_run_option_parser():
     r"""Test option parser in %run.
 
     In [1]: %run print_argv.py
     []
 
     In [2]: %run print_argv.py print*.py
     ['print_argv.py']
 
     In [3]: %run -G print_argv.py print*.py
     ['print*.py']
 
     """
 
 
 @dec.skip_win32
 def doctest_run_option_parser_for_posix():
     r"""Test option parser in %run (Linux/OSX specific).
 
     You need double quote to escape glob in POSIX systems:
 
     In [1]: %run print_argv.py print\\*.py
     ['print*.py']
 
     You can't use quote to escape glob in POSIX systems:
 
     In [2]: %run print_argv.py 'print*.py'
     ['print_argv.py']
 
     """
 
 
 @dec.skip_if_not_win32
 def doctest_run_option_parser_for_windows():
     r"""Test option parser in %run (Windows specific).
 
     In Windows, you can't escape ``*` `by backslash:
 
     In [1]: %run print_argv.py print\\*.py
     ['print\\*.py']
 
     You can use quote to escape glob:
 
     In [2]: %run print_argv.py 'print*.py'
     ['print*.py']
 
     """
 
 
 @py3compat.doctest_refactor_print
 def doctest_reset_del():
     """Test that resetting doesn't cause errors in __del__ methods.
 
     In [2]: class A(object):
        ...:     def __del__(self):
        ...:         print str("Hi")
        ...:
 
     In [3]: a = A()
 
     In [4]: get_ipython().reset()
     Hi
 
     In [5]: 1+1
     Out[5]: 2
     """
 
 # For some tests, it will be handy to organize them in a class with a common
 # setup that makes a temp file
 
 class TestMagicRunPass(tt.TempFileMixin):
 
     def setup(self):
         """Make a valid python temp file."""
         self.mktmp('pass\n')
         
     def run_tmpfile(self):
         _ip = get_ipython()
         # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
         # See below and ticket https://bugs.launchpad.net/bugs/366353
         _ip.magic('run %s' % self.fname)
         
     def run_tmpfile_p(self):
         _ip = get_ipython()
         # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
         # See below and ticket https://bugs.launchpad.net/bugs/366353
         _ip.magic('run -p %s' % self.fname)
 
     def test_builtins_id(self):
         """Check that %run doesn't damage __builtins__ """
         _ip = get_ipython()
         # Test that the id of __builtins__ is not modified by %run
         bid1 = id(_ip.user_ns['__builtins__'])
         self.run_tmpfile()
         bid2 = id(_ip.user_ns['__builtins__'])
         nt.assert_equal(bid1, bid2)
 
     def test_builtins_type(self):
         """Check that the type of __builtins__ doesn't change with %run.
 
         However, the above could pass if __builtins__ was already modified to
         be a dict (it should be a module) by a previous use of %run.  So we
         also check explicitly that it really is a module:
         """
         _ip = get_ipython()
         self.run_tmpfile()
         nt.assert_equal(type(_ip.user_ns['__builtins__']),type(sys))
 
     def test_prompts(self):
         """Test that prompts correctly generate after %run"""
         self.run_tmpfile()
         _ip = get_ipython()
         p2 = _ip.prompt_manager.render('in2').strip()
         nt.assert_equal(p2[:3], '...')
         
     def test_run_profile( self ):
         """Test that the option -p, which invokes the profiler, do not
         crash by invoking execfile"""
         _ip = get_ipython()
         self.run_tmpfile_p()
 
 
 class TestMagicRunSimple(tt.TempFileMixin):
 
     def test_simpledef(self):
         """Test that simple class definitions work."""
         src = ("class foo: pass\n"
                "def f(): return foo()")
         self.mktmp(src)
         _ip.magic('run %s' % self.fname)
         _ip.run_cell('t = isinstance(f(), foo)')
         nt.assert_true(_ip.user_ns['t'])
 
     def test_obj_del(self):
         """Test that object's __del__ methods are called on exit."""
         if sys.platform == 'win32':
             try:
                 import win32api
             except ImportError:
                 raise SkipTest("Test requires pywin32")
         src = ("class A(object):\n"
                "    def __del__(self):\n"
                "        print 'object A deleted'\n"
                "a = A()\n")
         self.mktmp(py3compat.doctest_refactor_print(src))
         if dec.module_not_available('sqlite3'):
             err = 'WARNING: IPython History requires SQLite, your history will not be saved\n'
         else:
             err = None
         tt.ipexec_validate(self.fname, 'object A deleted', err)
     
     def test_aggressive_namespace_cleanup(self):
         """Test that namespace cleanup is not too aggressive GH-238
 
         Returning from another run magic deletes the namespace"""
         # see ticket https://github.com/ipython/ipython/issues/238
         class secondtmp(tt.TempFileMixin): pass
         empty = secondtmp()
         empty.mktmp('')
         # On Windows, the filename will have \users in it, so we need to use the
         # repr so that the \u becomes \\u.
         src = ("ip = get_ipython()\n"
                "for i in range(5):\n"
                "   try:\n"
                "       ip.magic(%r)\n"
                "   except NameError as e:\n"
                "       print(i)\n"
                "       break\n" % ('run ' + empty.fname))
         self.mktmp(src)
         _ip.magic('run %s' % self.fname)
         _ip.run_cell('ip == get_ipython()')
         nt.assert_equal(_ip.user_ns['i'], 4)
     
     def test_run_second(self):
         """Test that running a second file doesn't clobber the first, gh-3547
         """
         self.mktmp("avar = 1\n"
                    "def afunc():\n"
                    "  return avar\n")
 
         empty = tt.TempFileMixin()
         empty.mktmp("")
         
         _ip.magic('run %s' % self.fname)
         _ip.magic('run %s' % empty.fname)
         nt.assert_equal(_ip.user_ns['afunc'](), 1)
 
     @dec.skip_win32
     def test_tclass(self):
         mydir = os.path.dirname(__file__)
         tc = os.path.join(mydir, 'tclass')
         src = ("%%run '%s' C-first\n"
                "%%run '%s' C-second\n"
                "%%run '%s' C-third\n") % (tc, tc, tc)
         self.mktmp(src, '.ipy')
         out = """\
 ARGV 1-: ['C-first']
 ARGV 1-: ['C-second']
 tclass.py: deleting object: C-first
 ARGV 1-: ['C-third']
 tclass.py: deleting object: C-second
 tclass.py: deleting object: C-third
 """
         if dec.module_not_available('sqlite3'):
             err = 'WARNING: IPython History requires SQLite, your history will not be saved\n'
         else:
             err = None
         tt.ipexec_validate(self.fname, out, err)
 
     def test_run_i_after_reset(self):
         """Check that %run -i still works after %reset (gh-693)"""
         src = "yy = zz\n"
         self.mktmp(src)
         _ip.run_cell("zz = 23")
         _ip.magic('run -i %s' % self.fname)
         nt.assert_equal(_ip.user_ns['yy'], 23)
         _ip.magic('reset -f')
         _ip.run_cell("zz = 23")
         _ip.magic('run -i %s' % self.fname)
         nt.assert_equal(_ip.user_ns['yy'], 23)
     
     def test_unicode(self):
         """Check that files in odd encodings are accepted."""
         mydir = os.path.dirname(__file__)
         na = os.path.join(mydir, 'nonascii.py')
         _ip.magic('run "%s"' % na)
         nt.assert_equal(_ip.user_ns['u'], u'Ўт№Ф')
 
     def test_run_py_file_attribute(self):
         """Test handling of `__file__` attribute in `%run <file>.py`."""
         src = "t = __file__\n"
         self.mktmp(src)
         _missing = object()
         file1 = _ip.user_ns.get('__file__', _missing)
         _ip.magic('run %s' % self.fname)
         file2 = _ip.user_ns.get('__file__', _missing)
 
         # Check that __file__ was equal to the filename in the script's
         # namespace.
         nt.assert_equal(_ip.user_ns['t'], self.fname)
 
         # Check that __file__ was not leaked back into user_ns.
         nt.assert_equal(file1, file2)
 
     def test_run_ipy_file_attribute(self):
         """Test handling of `__file__` attribute in `%run <file.ipy>`."""
         src = "t = __file__\n"
         self.mktmp(src, ext='.ipy')
         _missing = object()
         file1 = _ip.user_ns.get('__file__', _missing)
         _ip.magic('run %s' % self.fname)
         file2 = _ip.user_ns.get('__file__', _missing)
 
         # Check that __file__ was equal to the filename in the script's
         # namespace.
         nt.assert_equal(_ip.user_ns['t'], self.fname)
 
         # Check that __file__ was not leaked back into user_ns.
         nt.assert_equal(file1, file2)
 
     def test_run_formatting(self):
         """ Test that %run -t -N<N> does not raise a TypeError for N > 1."""
         src = "pass"
         self.mktmp(src)
         _ip.magic('run -t -N 1 %s' % self.fname)
         _ip.magic('run -t -N 10 %s' % self.fname)
     
     def test_ignore_sys_exit(self):
         """Test the -e option to ignore sys.exit()"""
         src = "import sys; sys.exit(1)"
         self.mktmp(src)
         with tt.AssertPrints('SystemExit'):
             _ip.magic('run %s' % self.fname)
         
         with tt.AssertNotPrints('SystemExit'):
             _ip.magic('run -e %s' % self.fname)
         
 
 
 class TestMagicRunWithPackage(unittest.TestCase):
 
     def writefile(self, name, content):
         path = os.path.join(self.tempdir.name, name)
         d = os.path.dirname(path)
         if not os.path.isdir(d):
             os.makedirs(d)
         with open(path, 'w') as f:
             f.write(textwrap.dedent(content))
 
     def setUp(self):
         self.package = package = 'tmp{0}'.format(repr(random.random())[2:])
         """Temporary valid python package name."""
 
         self.value = int(random.random() * 10000)
 
         self.tempdir = TemporaryDirectory()
-        self.__orig_cwd = os.getcwdu()
+        self.__orig_cwd = py3compat.getcwd()
         sys.path.insert(0, self.tempdir.name)
 
         self.writefile(os.path.join(package, '__init__.py'), '')
         self.writefile(os.path.join(package, 'sub.py'), """
         x = {0!r}
         """.format(self.value))
         self.writefile(os.path.join(package, 'relative.py'), """
         from .sub import x
         """)
         self.writefile(os.path.join(package, 'absolute.py'), """
         from {0}.sub import x
         """.format(package))
 
     def tearDown(self):
         os.chdir(self.__orig_cwd)
         sys.path[:] = [p for p in sys.path if p != self.tempdir.name]
         self.tempdir.cleanup()
 
     def check_run_submodule(self, submodule, opts=''):
         _ip.user_ns.pop('x', None)
         _ip.magic('run {2} -m {0}.{1}'.format(self.package, submodule, opts))
         self.assertEqual(_ip.user_ns['x'], self.value,
                          'Variable `x` is not loaded from module `{0}`.'
                          .format(submodule))
 
     def test_run_submodule_with_absolute_import(self):
         self.check_run_submodule('absolute')
 
     def test_run_submodule_with_relative_import(self):
         """Run submodule that has a relative import statement (#2727)."""
         self.check_run_submodule('relative')
 
     def test_prun_submodule_with_absolute_import(self):
         self.check_run_submodule('absolute', '-p')
 
     def test_prun_submodule_with_relative_import(self):
         self.check_run_submodule('relative', '-p')
 
     def with_fake_debugger(func):
         @functools.wraps(func)
         def wrapper(*args, **kwds):
             with tt.monkeypatch(debugger.Pdb, 'run', staticmethod(eval)):
                 return func(*args, **kwds)
         return wrapper
 
     @with_fake_debugger
     def test_debug_run_submodule_with_absolute_import(self):
         self.check_run_submodule('absolute', '-d')
 
     @with_fake_debugger
     def test_debug_run_submodule_with_relative_import(self):
         self.check_run_submodule('relative', '-d')
 
 def test_run__name__():
     with TemporaryDirectory() as td:
         path = pjoin(td, 'foo.py')
         with open(path, 'w') as f:
             f.write("q = __name__")
         
         _ip.user_ns.pop('q', None)
         _ip.magic('run {}'.format(path))
         nt.assert_equal(_ip.user_ns.pop('q'), '__main__')
         
         _ip.magic('run -n {}'.format(path))
         nt.assert_equal(_ip.user_ns.pop('q'), 'foo')

IPython/external/path/_path.py

@@ -1,1267 +1,1267 b''
 #
 # Copyright (c) 2010 Mikhail Gusarov
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 #
 
 """ path.py - An object representing a path to a file or directory.
 
 Original author:
  Jason Orendorff <jason.orendorff\x40gmail\x2ecom>
 
 Current maintainer:
  Jason R. Coombs <[e-mail]>
 
 Contributors:
  Mikhail Gusarov <[e-mail]>
  Marc Abramowitz <[e-mail]>
  Jason R. Coombs <[e-mail]>
  Jason Chu <[e-mail]>
  Vojislav Stojkovic <[e-mail]>
 
 Example::
 
     from path import path
     d = path('/home/guido/bin')
     for f in d.files('*.py'):
         f.chmod(0755)
 
 path.py requires Python 2.5 or later.
 """
 
 from __future__ import with_statement
 
 import sys
 import warnings
 import os
 import fnmatch
 import glob
 import shutil
 import codecs
 import hashlib
 import errno
 import tempfile
 import functools
 import operator
 import re
 
 try:
     import win32security
 except ImportError:
     pass
 
 try:
     import pwd
 except ImportError:
     pass
 
 ################################
 # Monkey patchy python 3 support
 try:
     basestring
 except NameError:
     basestring = str
 
 try:
     unicode
 except NameError:
     unicode = str
 
 try:
-    os.getcwdu
+    getcwdu = os.getcwdu
 except AttributeError:
-    os.getcwdu = os.getcwd
+    getcwdu = os.getcwd
 
 if sys.version < '3':
     def u(x):
         return codecs.unicode_escape_decode(x)[0]
 else:
     def u(x):
         return x
 
 o777 = 511
 o766 = 502
 o666 = 438
 o554 = 364
 ################################
 
 __version__ = '4.3'
 __all__ = ['path']
 
 
 class TreeWalkWarning(Warning):
     pass
 
 
 def simple_cache(func):
     """
     Save results for the 'using_module' classmethod.
     When Python 3.2 is available, use functools.lru_cache instead.
     """
     saved_results = {}
 
     def wrapper(cls, module):
         if module in saved_results:
             return saved_results[module]
         saved_results[module] = func(cls, module)
         return saved_results[module]
     return wrapper
 
 
 class ClassProperty(property):
     def __get__(self, cls, owner):
         return self.fget.__get__(None, owner)()
 
 
 class multimethod(object):
     """
     Acts like a classmethod when invoked from the class and like an
     instancemethod when invoked from the instance.
     """
     def __init__(self, func):
         self.func = func
 
     def __get__(self, instance, owner):
         return (
             functools.partial(self.func, owner) if instance is None
             else functools.partial(self.func, owner, instance)
         )
 
 
 class path(unicode):
     """ Represents a filesystem path.
 
     For documentation on individual methods, consult their
     counterparts in os.path.
     """
 
     module = os.path
     "The path module to use for path operations."
 
     def __init__(self, other=''):
         if other is None:
             raise TypeError("Invalid initial value for path: None")
 
     @classmethod
     @simple_cache
     def using_module(cls, module):
         subclass_name = cls.__name__ + '_' + module.__name__
         bases = (cls,)
         ns = {'module': module}
         return type(subclass_name, bases, ns)
 
     @ClassProperty
     @classmethod
     def _next_class(cls):
         """
         What class should be used to construct new instances from this class
         """
         return cls
 
     # --- Special Python methods.
 
     def __repr__(self):
         return '%s(%s)' % (type(self).__name__, super(path, self).__repr__())
 
     # Adding a path and a string yields a path.
     def __add__(self, more):
         try:
             return self._next_class(super(path, self).__add__(more))
         except TypeError:  # Python bug
             return NotImplemented
 
     def __radd__(self, other):
         if not isinstance(other, basestring):
             return NotImplemented
         return self._next_class(other.__add__(self))
 
     # The / operator joins paths.
     def __div__(self, rel):
         """ fp.__div__(rel) == fp / rel == fp.joinpath(rel)
 
         Join two path components, adding a separator character if
         needed.
         """
         return self._next_class(self.module.join(self, rel))
 
     # Make the / operator work even when true division is enabled.
     __truediv__ = __div__
 
     def __enter__(self):
         self._old_dir = self.getcwd()
         os.chdir(self)
         return self
 
     def __exit__(self, *_):
         os.chdir(self._old_dir)
 
     @classmethod
     def getcwd(cls):
         """ Return the current working directory as a path object. """
-        return cls(os.getcwdu())
+        return cls(getcwdu())
 
     #
     # --- Operations on path strings.
 
     def abspath(self):
         return self._next_class(self.module.abspath(self))
 
     def normcase(self):
         return self._next_class(self.module.normcase(self))
 
     def normpath(self):
         return self._next_class(self.module.normpath(self))
 
     def realpath(self):
         return self._next_class(self.module.realpath(self))
 
     def expanduser(self):
         return self._next_class(self.module.expanduser(self))
 
     def expandvars(self):
         return self._next_class(self.module.expandvars(self))
 
     def dirname(self):
         return self._next_class(self.module.dirname(self))
 
     def basename(self):
         return self._next_class(self.module.basename(self))
 
     def expand(self):
         """ Clean up a filename by calling expandvars(),
         expanduser(), and normpath() on it.
 
         This is commonly everything needed to clean up a filename
         read from a configuration file, for example.
         """
         return self.expandvars().expanduser().normpath()
 
     @property
     def namebase(self):
         """ The same as path.name, but with one file extension stripped off.
 
         For example, path('/home/guido/python.tar.gz').name == 'python.tar.gz',
         but          path('/home/guido/python.tar.gz').namebase == 'python.tar'
         """
         base, ext = self.module.splitext(self.name)
         return base
 
     @property
     def ext(self):
         """ The file extension, for example '.py'. """
         f, ext = self.module.splitext(self)
         return ext
 
     @property
     def drive(self):
         """ The drive specifier, for example 'C:'.
         This is always empty on systems that don't use drive specifiers.
         """
         drive, r = self.module.splitdrive(self)
         return self._next_class(drive)
 
     parent = property(
         dirname, None, None,
         """ This path's parent directory, as a new path object.
 
         For example,
         path('/usr/local/lib/libpython.so').parent == path('/usr/local/lib')
         """)
 
     name = property(
         basename, None, None,
         """ The name of this file or directory without the full path.
 
         For example, path('/usr/local/lib/libpython.so').name == 'libpython.so'
         """)
 
     def splitpath(self):
         """ p.splitpath() -> Return (p.parent, p.name). """
         parent, child = self.module.split(self)
         return self._next_class(parent), child
 
     def splitdrive(self):
         """ p.splitdrive() -> Return (p.drive, <the rest of p>).
 
         Split the drive specifier from this path.  If there is
         no drive specifier, p.drive is empty, so the return value
         is simply (path(''), p).  This is always the case on Unix.
         """
         drive, rel = self.module.splitdrive(self)
         return self._next_class(drive), rel
 
     def splitext(self):
         """ p.splitext() -> Return (p.stripext(), p.ext).
 
         Split the filename extension from this path and return
         the two parts.  Either part may be empty.
 
         The extension is everything from '.' to the end of the
         last path segment.  This has the property that if
         (a, b) == p.splitext(), then a + b == p.
         """
         filename, ext = self.module.splitext(self)
         return self._next_class(filename), ext
 
     def stripext(self):
         """ p.stripext() -> Remove one file extension from the path.
 
         For example, path('/home/guido/python.tar.gz').stripext()
         returns path('/home/guido/python.tar').
         """
         return self.splitext()[0]
 
     def splitunc(self):
         unc, rest = self.module.splitunc(self)
         return self._next_class(unc), rest
 
     @property
     def uncshare(self):
         """
         The UNC mount point for this path.
         This is empty for paths on local drives.
         """
         unc, r = self.module.splitunc(self)
         return self._next_class(unc)
 
     @multimethod
     def joinpath(cls, first, *others):
         """
         Join first to zero or more path components, adding a separator
         character (first.module.sep) if needed.  Returns a new instance of
         first._next_class.
         """
         if not isinstance(first, cls):
             first = cls(first)
         return first._next_class(first.module.join(first, *others))
 
     def splitall(self):
         r""" Return a list of the path components in this path.
 
         The first item in the list will be a path.  Its value will be
         either os.curdir, os.pardir, empty, or the root directory of
         this path (for example, ``'/'`` or ``'C:\\'``).  The other items in
         the list will be strings.
 
         ``path.path.joinpath(*result)`` will yield the original path.
         """
         parts = []
         loc = self
         while loc != os.curdir and loc != os.pardir:
             prev = loc
             loc, child = prev.splitpath()
             if loc == prev:
                 break
             parts.append(child)
         parts.append(loc)
         parts.reverse()
         return parts
 
     def relpath(self, start='.'):
         """ Return this path as a relative path,
         based from start, which defaults to the current working directory.
         """
         cwd = self._next_class(start)
         return cwd.relpathto(self)
 
     def relpathto(self, dest):
         """ Return a relative path from self to dest.
 
         If there is no relative path from self to dest, for example if
         they reside on different drives in Windows, then this returns
         dest.abspath().
         """
         origin = self.abspath()
         dest = self._next_class(dest).abspath()
 
         orig_list = origin.normcase().splitall()
         # Don't normcase dest!  We want to preserve the case.
         dest_list = dest.splitall()
 
         if orig_list[0] != self.module.normcase(dest_list[0]):
             # Can't get here from there.
             return dest
 
         # Find the location where the two paths start to differ.
         i = 0
         for start_seg, dest_seg in zip(orig_list, dest_list):
             if start_seg != self.module.normcase(dest_seg):
                 break
             i += 1
 
         # Now i is the point where the two paths diverge.
         # Need a certain number of "os.pardir"s to work up
         # from the origin to the point of divergence.
         segments = [os.pardir] * (len(orig_list) - i)
         # Need to add the diverging part of dest_list.
         segments += dest_list[i:]
         if len(segments) == 0:
             # If they happen to be identical, use os.curdir.
             relpath = os.curdir
         else:
             relpath = self.module.join(*segments)
         return self._next_class(relpath)
 
     # --- Listing, searching, walking, and matching
 
     def listdir(self, pattern=None):
         """ D.listdir() -> List of items in this directory.
 
         Use D.files() or D.dirs() instead if you want a listing
         of just files or just subdirectories.
 
         The elements of the list are path objects.
 
         With the optional 'pattern' argument, this only lists
         items whose names match the given pattern.
         """
         names = os.listdir(self)
         if pattern is not None:
             names = fnmatch.filter(names, pattern)
         return [self / child for child in names]
 
     def dirs(self, pattern=None):
         """ D.dirs() -> List of this directory's subdirectories.
 
         The elements of the list are path objects.
         This does not walk recursively into subdirectories
         (but see path.walkdirs).
 
         With the optional 'pattern' argument, this only lists
         directories whose names match the given pattern.  For
         example, ``d.dirs('build-*')``.
         """
         return [p for p in self.listdir(pattern) if p.isdir()]
 
     def files(self, pattern=None):
         """ D.files() -> List of the files in this directory.
 
         The elements of the list are path objects.
         This does not walk into subdirectories (see path.walkfiles).
 
         With the optional 'pattern' argument, this only lists files
         whose names match the given pattern.  For example,
         ``d.files('*.pyc')``.
         """
 
         return [p for p in self.listdir(pattern) if p.isfile()]
 
     def walk(self, pattern=None, errors='strict'):
         """ D.walk() -> iterator over files and subdirs, recursively.
 
         The iterator yields path objects naming each child item of
         this directory and its descendants.  This requires that
         D.isdir().
 
         This performs a depth-first traversal of the directory tree.
         Each directory is returned just before all its children.
 
         The errors= keyword argument controls behavior when an
         error occurs.  The default is 'strict', which causes an
         exception.  The other allowed values are 'warn', which
         reports the error via warnings.warn(), and 'ignore'.
         """
         if errors not in ('strict', 'warn', 'ignore'):
             raise ValueError("invalid errors parameter")
 
         try:
             childList = self.listdir()
         except Exception:
             if errors == 'ignore':
                 return
             elif errors == 'warn':
                 warnings.warn(
                     "Unable to list directory '%s': %s"
                     % (self, sys.exc_info()[1]),
                     TreeWalkWarning)
                 return
             else:
                 raise
 
         for child in childList:
             if pattern is None or child.fnmatch(pattern):
                 yield child
             try:
                 isdir = child.isdir()
             except Exception:
                 if errors == 'ignore':
                     isdir = False
                 elif errors == 'warn':
                     warnings.warn(
                         "Unable to access '%s': %s"
                         % (child, sys.exc_info()[1]),
                         TreeWalkWarning)
                     isdir = False
                 else:
                     raise
 
             if isdir:
                 for item in child.walk(pattern, errors):
                     yield item
 
     def walkdirs(self, pattern=None, errors='strict'):
         """ D.walkdirs() -> iterator over subdirs, recursively.
 
         With the optional 'pattern' argument, this yields only
         directories whose names match the given pattern.  For
         example, ``mydir.walkdirs('*test')`` yields only directories
         with names ending in 'test'.
 
         The errors= keyword argument controls behavior when an
         error occurs.  The default is 'strict', which causes an
         exception.  The other allowed values are 'warn', which
         reports the error via warnings.warn(), and 'ignore'.
         """
         if errors not in ('strict', 'warn', 'ignore'):
             raise ValueError("invalid errors parameter")
 
         try:
             dirs = self.dirs()
         except Exception:
             if errors == 'ignore':
                 return
             elif errors == 'warn':
                 warnings.warn(
                     "Unable to list directory '%s': %s"
                     % (self, sys.exc_info()[1]),
                     TreeWalkWarning)
                 return
             else:
                 raise
 
         for child in dirs:
             if pattern is None or child.fnmatch(pattern):
                 yield child
             for subsubdir in child.walkdirs(pattern, errors):
                 yield subsubdir
 
     def walkfiles(self, pattern=None, errors='strict'):
         """ D.walkfiles() -> iterator over files in D, recursively.
 
         The optional argument, pattern, limits the results to files
         with names that match the pattern.  For example,
         ``mydir.walkfiles('*.tmp')`` yields only files with the .tmp
         extension.
         """
         if errors not in ('strict', 'warn', 'ignore'):
             raise ValueError("invalid errors parameter")
 
         try:
             childList = self.listdir()
         except Exception:
             if errors == 'ignore':
                 return
             elif errors == 'warn':
                 warnings.warn(
                     "Unable to list directory '%s': %s"
                     % (self, sys.exc_info()[1]),
                     TreeWalkWarning)
                 return
             else:
                 raise
 
         for child in childList:
             try:
                 isfile = child.isfile()
                 isdir = not isfile and child.isdir()
             except:
                 if errors == 'ignore':
                     continue
                 elif errors == 'warn':
                     warnings.warn(
                         "Unable to access '%s': %s"
                         % (self, sys.exc_info()[1]),
                         TreeWalkWarning)
                     continue
                 else:
                     raise
 
             if isfile:
                 if pattern is None or child.fnmatch(pattern):
                     yield child
             elif isdir:
                 for f in child.walkfiles(pattern, errors):
                     yield f
 
     def fnmatch(self, pattern):
         """ Return True if self.name matches the given pattern.
 
         pattern - A filename pattern with wildcards,
             for example ``'*.py'``.
         """
         return fnmatch.fnmatch(self.name, pattern)
 
     def glob(self, pattern):
         """ Return a list of path objects that match the pattern.
 
         pattern - a path relative to this directory, with wildcards.
 
         For example, path('/users').glob('*/bin/*') returns a list
         of all the files users have in their bin directories.
         """
         cls = self._next_class
         return [cls(s) for s in glob.glob(self / pattern)]
 
     #
     # --- Reading or writing an entire file at once.
 
     def open(self, *args, **kwargs):
         """ Open this file.  Return a file object. """
         return open(self, *args, **kwargs)
 
     def bytes(self):
         """ Open this file, read all bytes, return them as a string. """
         with self.open('rb') as f:
             return f.read()
 
     def chunks(self, size, *args, **kwargs):
         """ Returns a generator yielding chunks of the file, so it can
             be read piece by piece with a simple for loop.
 
            Any argument you pass after `size` will be passed to `open()`.
 
            :example:
 
                >>> for chunk in path("file.txt").chunk(8192):
                ...    print(chunk)
 
             This will read the file by chunks of 8192 bytes.
         """
         with open(self, *args, **kwargs) as f:
             while True:
                 d = f.read(size)
                 if not d:
                     break
                 yield d
 
     def write_bytes(self, bytes, append=False):
         """ Open this file and write the given bytes to it.
 
         Default behavior is to overwrite any existing file.
         Call p.write_bytes(bytes, append=True) to append instead.
         """
         if append:
             mode = 'ab'
         else:
             mode = 'wb'
         with self.open(mode) as f:
             f.write(bytes)
 
     def text(self, encoding=None, errors='strict'):
         r""" Open this file, read it in, return the content as a string.
 
         This method uses 'U' mode, so '\r\n' and '\r' are automatically
         translated to '\n'.
 
         Optional arguments:
 
         encoding - The Unicode encoding (or character set) of
             the file.  If present, the content of the file is
             decoded and returned as a unicode object; otherwise
             it is returned as an 8-bit str.
         errors - How to handle Unicode errors; see help(str.decode)
             for the options.  Default is 'strict'.
         """
         if encoding is None:
             # 8-bit
             with self.open('U') as f:
                 return f.read()
         else:
             # Unicode
             with codecs.open(self, 'r', encoding, errors) as f:
                 # (Note - Can't use 'U' mode here, since codecs.open
                 # doesn't support 'U' mode.)
                 t = f.read()
             return (t.replace(u('\r\n'), u('\n'))
                      .replace(u('\r\x85'), u('\n'))
                      .replace(u('\r'), u('\n'))
                      .replace(u('\x85'), u('\n'))
                      .replace(u('\u2028'), u('\n')))
 
     def write_text(self, text, encoding=None, errors='strict',
                    linesep=os.linesep, append=False):
         r""" Write the given text to this file.
 
         The default behavior is to overwrite any existing file;
         to append instead, use the 'append=True' keyword argument.
 
         There are two differences between path.write_text() and
         path.write_bytes(): newline handling and Unicode handling.
         See below.
 
         Parameters:
 
           - text - str/unicode - The text to be written.
 
           - encoding - str - The Unicode encoding that will be used.
             This is ignored if 'text' isn't a Unicode string.
 
           - errors - str - How to handle Unicode encoding errors.
             Default is 'strict'.  See help(unicode.encode) for the
             options.  This is ignored if 'text' isn't a Unicode
             string.
 
           - linesep - keyword argument - str/unicode - The sequence of
             characters to be used to mark end-of-line.  The default is
             os.linesep.  You can also specify None; this means to
             leave all newlines as they are in 'text'.
 
           - append - keyword argument - bool - Specifies what to do if
             the file already exists (True: append to the end of it;
             False: overwrite it.)  The default is False.
 
 
         --- Newline handling.
 
         write_text() converts all standard end-of-line sequences
         ('\n', '\r', and '\r\n') to your platform's default end-of-line
         sequence (see os.linesep; on Windows, for example, the
         end-of-line marker is '\r\n').
 
         If you don't like your platform's default, you can override it
         using the 'linesep=' keyword argument.  If you specifically want
         write_text() to preserve the newlines as-is, use 'linesep=None'.
 
         This applies to Unicode text the same as to 8-bit text, except
         there are three additional standard Unicode end-of-line sequences:
         u'\x85', u'\r\x85', and u'\u2028'.
 
         (This is slightly different from when you open a file for
         writing with fopen(filename, "w") in C or open(filename, 'w')
         in Python.)
 
 
         --- Unicode
 
         If 'text' isn't Unicode, then apart from newline handling, the
         bytes are written verbatim to the file.  The 'encoding' and
         'errors' arguments are not used and must be omitted.
 
         If 'text' is Unicode, it is first converted to bytes using the
         specified 'encoding' (or the default encoding if 'encoding'
         isn't specified).  The 'errors' argument applies only to this
         conversion.
 
         """
         if isinstance(text, unicode):
             if linesep is not None:
                 # Convert all standard end-of-line sequences to
                 # ordinary newline characters.
                 text = (text.replace(u('\r\n'), u('\n'))
                             .replace(u('\r\x85'), u('\n'))
                             .replace(u('\r'), u('\n'))
                             .replace(u('\x85'), u('\n'))
                             .replace(u('\u2028'), u('\n')))
                 text = text.replace(u('\n'), linesep)
             if encoding is None:
                 encoding = sys.getdefaultencoding()
             bytes = text.encode(encoding, errors)
         else:
             # It is an error to specify an encoding if 'text' is
             # an 8-bit string.
             assert encoding is None
 
             if linesep is not None:
                 text = (text.replace('\r\n', '\n')
                             .replace('\r', '\n'))
                 bytes = text.replace('\n', linesep)
 
         self.write_bytes(bytes, append)
 
     def lines(self, encoding=None, errors='strict', retain=True):
         r""" Open this file, read all lines, return them in a list.
 
         Optional arguments:
             encoding - The Unicode encoding (or character set) of
                 the file.  The default is None, meaning the content
                 of the file is read as 8-bit characters and returned
                 as a list of (non-Unicode) str objects.
             errors - How to handle Unicode errors; see help(str.decode)
                 for the options.  Default is 'strict'
             retain - If true, retain newline characters; but all newline
                 character combinations ('\r', '\n', '\r\n') are
                 translated to '\n'.  If false, newline characters are
                 stripped off.  Default is True.
 
         This uses 'U' mode.
         """
         if encoding is None and retain:
             with self.open('U') as f:
                 return f.readlines()
         else:
             return self.text(encoding, errors).splitlines(retain)
 
     def write_lines(self, lines, encoding=None, errors='strict',
                     linesep=os.linesep, append=False):
         r""" Write the given lines of text to this file.
 
         By default this overwrites any existing file at this path.
 
         This puts a platform-specific newline sequence on every line.
         See 'linesep' below.
 
         lines - A list of strings.
 
         encoding - A Unicode encoding to use.  This applies only if
             'lines' contains any Unicode strings.
 
         errors - How to handle errors in Unicode encoding.  This
             also applies only to Unicode strings.
 
         linesep - The desired line-ending.  This line-ending is
             applied to every line.  If a line already has any
             standard line ending ('\r', '\n', '\r\n', u'\x85',
             u'\r\x85', u'\u2028'), that will be stripped off and
             this will be used instead.  The default is os.linesep,
             which is platform-dependent ('\r\n' on Windows, '\n' on
             Unix, etc.)  Specify None to write the lines as-is,
             like file.writelines().
 
         Use the keyword argument append=True to append lines to the
         file.  The default is to overwrite the file.  Warning:
         When you use this with Unicode data, if the encoding of the
         existing data in the file is different from the encoding
         you specify with the encoding= parameter, the result is
         mixed-encoding data, which can really confuse someone trying
         to read the file later.
         """
         if append:
             mode = 'ab'
         else:
             mode = 'wb'
         with self.open(mode) as f:
             for line in lines:
                 isUnicode = isinstance(line, unicode)
                 if linesep is not None:
                     # Strip off any existing line-end and add the
                     # specified linesep string.
                     if isUnicode:
                         if line[-2:] in (u('\r\n'), u('\x0d\x85')):
                             line = line[:-2]
                         elif line[-1:] in (u('\r'), u('\n'),
                                            u('\x85'), u('\u2028')):
                             line = line[:-1]
                     else:
                         if line[-2:] == '\r\n':
                             line = line[:-2]
                         elif line[-1:] in ('\r', '\n'):
                             line = line[:-1]
                     line += linesep
                 if isUnicode:
                     if encoding is None:
                         encoding = sys.getdefaultencoding()
                     line = line.encode(encoding, errors)
                 f.write(line)
 
     def read_md5(self):
         """ Calculate the md5 hash for this file.
 
         This reads through the entire file.
         """
         return self.read_hash('md5')
 
     def _hash(self, hash_name):
         """ Returns a hash object for the file at the current path.
 
             `hash_name` should be a hash algo name such as 'md5' or 'sha1'
             that's available in the `hashlib` module.
         """
         m = hashlib.new(hash_name)
         for chunk in self.chunks(8192):
             m.update(chunk)
         return m
 
     def read_hash(self, hash_name):
         """ Calculate given hash for this file.
 
         List of supported hashes can be obtained from hashlib package. This
         reads the entire file.
         """
         return self._hash(hash_name).digest()
 
     def read_hexhash(self, hash_name):
         """ Calculate given hash for this file, returning hexdigest.
 
         List of supported hashes can be obtained from hashlib package. This
         reads the entire file.
         """
         return self._hash(hash_name).hexdigest()
 
     # --- Methods for querying the filesystem.
     # N.B. On some platforms, the os.path functions may be implemented in C
     # (e.g. isdir on Windows, Python 3.2.2), and compiled functions don't get
     # bound. Playing it safe and wrapping them all in method calls.
 
     def isabs(self):
         return self.module.isabs(self)
 
     def exists(self):
         return self.module.exists(self)
 
     def isdir(self):
         return self.module.isdir(self)
 
     def isfile(self):
         return self.module.isfile(self)
 
     def islink(self):
         return self.module.islink(self)
 
     def ismount(self):
         return self.module.ismount(self)
 
     def samefile(self, other):
         return self.module.samefile(self, other)
 
     def getatime(self):
         return self.module.getatime(self)
 
     atime = property(
         getatime, None, None,
         """ Last access time of the file. """)
 
     def getmtime(self):
         return self.module.getmtime(self)
 
     mtime = property(
         getmtime, None, None,
         """ Last-modified time of the file. """)
 
     def getctime(self):
         return self.module.getctime(self)
 
     ctime = property(
         getctime, None, None,
         """ Creation time of the file. """)
 
     def getsize(self):
         return self.module.getsize(self)
 
     size = property(
         getsize, None, None,
         """ Size of the file, in bytes. """)
 
     if hasattr(os, 'access'):
         def access(self, mode):
             """ Return true if current user has access to this path.
 
             mode - One of the constants os.F_OK, os.R_OK, os.W_OK, os.X_OK
             """
             return os.access(self, mode)
 
     def stat(self):
         """ Perform a stat() system call on this path. """
         return os.stat(self)
 
     def lstat(self):
         """ Like path.stat(), but do not follow symbolic links. """
         return os.lstat(self)
 
     def __get_owner_windows(self):
         r"""
         Return the name of the owner of this file or directory. Follow
         symbolic links.
 
         Return a name of the form ur'DOMAIN\User Name'; may be a group.
         """
         desc = win32security.GetFileSecurity(
             self, win32security.OWNER_SECURITY_INFORMATION)
         sid = desc.GetSecurityDescriptorOwner()
         account, domain, typecode = win32security.LookupAccountSid(None, sid)
         return domain + u('\\') + account
 
     def __get_owner_unix(self):
         """
         Return the name of the owner of this file or directory. Follow
         symbolic links.
         """
         st = self.stat()
         return pwd.getpwuid(st.st_uid).pw_name
 
     def __get_owner_not_implemented(self):
         raise NotImplementedError("Ownership not available on this platform.")
 
     if 'win32security' in globals():
         get_owner = __get_owner_windows
     elif 'pwd' in globals():
         get_owner = __get_owner_unix
     else:
         get_owner = __get_owner_not_implemented
 
     owner = property(
         get_owner, None, None,
         """ Name of the owner of this file or directory. """)
 
     if hasattr(os, 'statvfs'):
         def statvfs(self):
             """ Perform a statvfs() system call on this path. """
             return os.statvfs(self)
 
     if hasattr(os, 'pathconf'):
         def pathconf(self, name):
             return os.pathconf(self, name)
 
     #
     # --- Modifying operations on files and directories
 
     def utime(self, times):
         """ Set the access and modified times of this file. """
         os.utime(self, times)
         return self
 
     def chmod(self, mode):
         os.chmod(self, mode)
         return self
 
     if hasattr(os, 'chown'):
         def chown(self, uid=-1, gid=-1):
             os.chown(self, uid, gid)
             return self
 
     def rename(self, new):
         os.rename(self, new)
         return self._next_class(new)
 
     def renames(self, new):
         os.renames(self, new)
         return self._next_class(new)
 
     #
     # --- Create/delete operations on directories
 
     def mkdir(self, mode=o777):
         os.mkdir(self, mode)
         return self
 
     def mkdir_p(self, mode=o777):
         try:
             self.mkdir(mode)
         except OSError:
             _, e, _ = sys.exc_info()
             if e.errno != errno.EEXIST:
                 raise
         return self
 
     def makedirs(self, mode=o777):
         os.makedirs(self, mode)
         return self
 
     def makedirs_p(self, mode=o777):
         try:
             self.makedirs(mode)
         except OSError:
             _, e, _ = sys.exc_info()
             if e.errno != errno.EEXIST:
                 raise
         return self
 
     def rmdir(self):
         os.rmdir(self)
         return self
 
     def rmdir_p(self):
         try:
             self.rmdir()
         except OSError:
             _, e, _ = sys.exc_info()
             if e.errno != errno.ENOTEMPTY and e.errno != errno.EEXIST:
                 raise
         return self
 
     def removedirs(self):
         os.removedirs(self)
         return self
 
     def removedirs_p(self):
         try:
             self.removedirs()
         except OSError:
             _, e, _ = sys.exc_info()
             if e.errno != errno.ENOTEMPTY and e.errno != errno.EEXIST:
                 raise
         return self
 
     # --- Modifying operations on files
 
     def touch(self):
         """ Set the access/modified times of this file to the current time.
         Create the file if it does not exist.
         """
         fd = os.open(self, os.O_WRONLY | os.O_CREAT, o666)
         os.close(fd)
         os.utime(self, None)
         return self
 
     def remove(self):
         os.remove(self)
         return self
 
     def remove_p(self):
         try:
             self.unlink()
         except OSError:
             _, e, _ = sys.exc_info()
             if e.errno != errno.ENOENT:
                 raise
         return self
 
     def unlink(self):
         os.unlink(self)
         return self
 
     def unlink_p(self):
         self.remove_p()
         return self
 
     # --- Links
 
     if hasattr(os, 'link'):
         def link(self, newpath):
             """ Create a hard link at 'newpath', pointing to this file. """
             os.link(self, newpath)
             return self._next_class(newpath)
 
     if hasattr(os, 'symlink'):
         def symlink(self, newlink):
             """ Create a symbolic link at 'newlink', pointing here. """
             os.symlink(self, newlink)
             return self._next_class(newlink)
 
     if hasattr(os, 'readlink'):
         def readlink(self):
             """ Return the path to which this symbolic link points.
 
             The result may be an absolute or a relative path.
             """
             return self._next_class(os.readlink(self))
 
         def readlinkabs(self):
             """ Return the path to which this symbolic link points.
 
             The result is always an absolute path.
             """
             p = self.readlink()
             if p.isabs():
                 return p
             else:
                 return (self.parent / p).abspath()
 
     #
     # --- High-level functions from shutil
 
     copyfile = shutil.copyfile
     copymode = shutil.copymode
     copystat = shutil.copystat
     copy = shutil.copy
     copy2 = shutil.copy2
     copytree = shutil.copytree
     if hasattr(shutil, 'move'):
         move = shutil.move
     rmtree = shutil.rmtree
 
     def rmtree_p(self):
         try:
             self.rmtree()
         except OSError:
             _, e, _ = sys.exc_info()
             if e.errno != errno.ENOENT:
                 raise
         return self
 
     def chdir(self):
         os.chdir(self)
 
     cd = chdir
 
     #
     # --- Special stuff from os
 
     if hasattr(os, 'chroot'):
         def chroot(self):
             os.chroot(self)
 
     if hasattr(os, 'startfile'):
         def startfile(self):
             os.startfile(self)
             return self
 
 
 class tempdir(path):
     """
     A temporary directory via tempfile.mkdtemp, and constructed with the
     same parameters that you can use as a context manager.
 
     Example:
 
         with tempdir() as d:
             # do stuff with the path object "d"
 
         # here the directory is deleted automatically
     """
 
     @ClassProperty
     @classmethod
     def _next_class(cls):
         return path
 
     def __new__(cls, *args, **kwargs):
         dirname = tempfile.mkdtemp(*args, **kwargs)
         return super(tempdir, cls).__new__(cls, dirname)
 
     def __init__(self, *args, **kwargs):
         pass
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
         if not exc_value:
             self.rmtree()
 
 
 def _permission_mask(mode):
     """
     Convert a Unix chmod symbolic mode like 'ugo+rwx' to a function
     suitable for applying to a mask to affect that change.
 
     >>> mask = _permission_mask('ugo+rwx')
     >>> oct(mask(o554))
     'o777'
 
     >>> oct(_permission_mask('gw-x')(o777))
     'o766'
     """
     parsed = re.match('(?P<who>[ugo]+)(?P<op>[-+])(?P<what>[rwx]+)$', mode)
     if not parsed:
         raise ValueError("Unrecognized symbolic mode", mode)
     spec_map = dict(r=4, w=2, x=1)
     spec = reduce(operator.or_, [spec_map[perm]
                   for perm in parsed.group('what')])
     # now apply spec to each in who
     shift_map = dict(u=6, g=3, o=0)
     mask = reduce(operator.or_, [spec << shift_map[subj]
                   for subj in parsed.group('who')])
 
     op = parsed.group('op')
     # if op is -, invert the mask
     if op == '-':
         mask ^= o777
 
     op_map = {'+': operator.or_, '-': operator.and_}
     return functools.partial(op_map[op], mask)

IPython/html/services/clusters/clustermanager.py

@@ -1,172 +1,171 b''
 """Manage IPython.parallel clusters in the notebook.
 
 Authors:
 
 * Brian Granger
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-import os
-
 from tornado import web
 from zmq.eventloop import ioloop
 
 from IPython.config.configurable import LoggingConfigurable
 from IPython.utils.traitlets import Dict, Instance, CFloat
 from IPython.parallel.apps.ipclusterapp import IPClusterStart
 from IPython.core.profileapp import list_profiles_in
 from IPython.core.profiledir import ProfileDir
+from IPython.utils import py3compat
 from IPython.utils.path import get_ipython_dir
 
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 
 class DummyIPClusterStart(IPClusterStart):
     """Dummy subclass to skip init steps that conflict with global app.
     
     Instantiating and initializing this class should result in fully configured
     launchers, but no other side effects or state.
     """
 
     def init_signal(self):
         pass
     def reinit_logging(self):
         pass
 
 
 class ClusterManager(LoggingConfigurable):
 
     profiles = Dict()
 
     delay = CFloat(1., config=True,
         help="delay (in s) between starting the controller and the engines")
 
     loop = Instance('zmq.eventloop.ioloop.IOLoop')
     def _loop_default(self):
         from zmq.eventloop.ioloop import IOLoop
         return IOLoop.instance()
 
     def build_launchers(self, profile_dir):
         starter = DummyIPClusterStart(log=self.log)
         starter.initialize(['--profile-dir', profile_dir])
         cl = starter.controller_launcher
         esl = starter.engine_launcher
         n = starter.n
         return cl, esl, n
 
     def get_profile_dir(self, name, path):
         p = ProfileDir.find_profile_dir_by_name(path,name=name)
         return p.location
 
     def update_profiles(self):
         """List all profiles in the ipython_dir and cwd.
         """
-        for path in [get_ipython_dir(), os.getcwdu()]:
+        for path in [get_ipython_dir(), py3compat.getcwd()]:
             for profile in list_profiles_in(path):
                 pd = self.get_profile_dir(profile, path)
                 if profile not in self.profiles:
                     self.log.debug("Adding cluster profile '%s'" % profile)
                     self.profiles[profile] = {
                         'profile': profile,
                         'profile_dir': pd,
                         'status': 'stopped'
                     }
 
     def list_profiles(self):
         self.update_profiles()
         # sorted list, but ensure that 'default' always comes first
         default_first = lambda name: name if name != 'default' else ''
         result = [self.profile_info(p) for p in sorted(self.profiles, key=default_first)]
         return result
 
     def check_profile(self, profile):
         if profile not in self.profiles:
             raise web.HTTPError(404, u'profile not found')
 
     def profile_info(self, profile):
         self.check_profile(profile)
         result = {}
         data = self.profiles.get(profile)
         result['profile'] = profile
         result['profile_dir'] = data['profile_dir']
         result['status'] = data['status']
         if 'n' in data:
             result['n'] = data['n']
         return result
 
     def start_cluster(self, profile, n=None):
         """Start a cluster for a given profile."""
         self.check_profile(profile)
         data = self.profiles[profile]
         if data['status'] == 'running':
             raise web.HTTPError(409, u'cluster already running')
         cl, esl, default_n = self.build_launchers(data['profile_dir'])
         n = n if n is not None else default_n
         def clean_data():
             data.pop('controller_launcher',None)
             data.pop('engine_set_launcher',None)
             data.pop('n',None)
             data['status'] = 'stopped'
         def engines_stopped(r):
             self.log.debug('Engines stopped')
             if cl.running:
                 cl.stop()
             clean_data()
         esl.on_stop(engines_stopped)
         def controller_stopped(r):
             self.log.debug('Controller stopped')
             if esl.running:
                 esl.stop()
             clean_data()
         cl.on_stop(controller_stopped)
 
         dc = ioloop.DelayedCallback(lambda: cl.start(), 0, self.loop)
         dc.start()
         dc = ioloop.DelayedCallback(lambda: esl.start(n), 1000*self.delay, self.loop)
         dc.start()
 
         self.log.debug('Cluster started')
         data['controller_launcher'] = cl
         data['engine_set_launcher'] = esl
         data['n'] = n
         data['status'] = 'running'
         return self.profile_info(profile)
 
     def stop_cluster(self, profile):
         """Stop a cluster for a given profile."""
         self.check_profile(profile)
         data = self.profiles[profile]
         if data['status'] == 'stopped':
             raise web.HTTPError(409, u'cluster not running')
         data = self.profiles[profile]
         cl = data['controller_launcher']
         esl = data['engine_set_launcher']
         if cl.running:
             cl.stop()
         if esl.running:
             esl.stop()
         # Return a temp info dict, the real one is updated in the on_stop
         # logic above.
         result = {
             'profile': data['profile'],
             'profile_dir': data['profile_dir'],
             'status': 'stopped'
         }
         return result
 
     def stop_all_clusters(self):
         for p in self.profiles.keys():
             self.stop_cluster(p)

IPython/html/services/notebooks/nbmanager.py

@@ -1,173 +1,174 b''
 """A base class notebook manager.
 
 Authors:
 
 * Brian Granger
 * Zach Sailer
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 
 from IPython.config.configurable import LoggingConfigurable
 from IPython.nbformat import current
-from IPython.utils.traitlets import List, Dict, Unicode, TraitError
+from IPython.utils import py3compat
+from IPython.utils.traitlets import Unicode, TraitError
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 class NotebookManager(LoggingConfigurable):
 
     # Todo:
     # The notebook_dir attribute is used to mean a couple of different things:
     # 1. Where the notebooks are stored if FileNotebookManager is used.
     # 2. The cwd of the kernel for a project.
     # Right now we use this attribute in a number of different places and
     # we are going to have to disentangle all of this.
-    notebook_dir = Unicode(os.getcwdu(), config=True, help="""
+    notebook_dir = Unicode(py3compat.getcwd(), config=True, help="""
             The directory to use for notebooks.
             """)
 
     filename_ext = Unicode(u'.ipynb')
     
     def path_exists(self, path):
         """Does the API-style path (directory) actually exist?
         
         Override this method in subclasses.
         
         Parameters
         ----------
         path : string
             The 
         
         Returns
         -------
         exists : bool
             Whether the path does indeed exist.
         """
         raise NotImplementedError
     
     def _notebook_dir_changed(self, name, old, new):
         """Do a bit of validation of the notebook dir."""
         if not os.path.isabs(new):
             # If we receive a non-absolute path, make it absolute.
             self.notebook_dir = os.path.abspath(new)
             return
         if os.path.exists(new) and not os.path.isdir(new):
             raise TraitError("notebook dir %r is not a directory" % new)
         if not os.path.exists(new):
             self.log.info("Creating notebook dir %s", new)
             try:
                 os.mkdir(new)
             except:
                 raise TraitError("Couldn't create notebook dir %r" % new)
 
     # Main notebook API
 
     def increment_filename(self, basename, path=''):
         """Increment a notebook filename without the .ipynb to make it unique.
         
         Parameters
         ----------
         basename : unicode
             The name of a notebook without the ``.ipynb`` file extension.
         path : unicode
             The URL path of the notebooks directory
         """
         return basename
 
     def list_notebooks(self, path=''):
         """Return a list of notebook dicts without content.
 
         This returns a list of dicts, each of the form::
 
             dict(notebook_id=notebook,name=name)
 
         This list of dicts should be sorted by name::
 
             data = sorted(data, key=lambda item: item['name'])
         """
         raise NotImplementedError('must be implemented in a subclass')
 
     def get_notebook_model(self, name, path='', content=True):
         """Get the notebook model with or without content."""
         raise NotImplementedError('must be implemented in a subclass')
 
     def save_notebook_model(self, model, name, path=''):
         """Save the notebook model and return the model with no content."""
         raise NotImplementedError('must be implemented in a subclass')
 
     def update_notebook_model(self, model, name, path=''):
         """Update the notebook model and return the model with no content."""
         raise NotImplementedError('must be implemented in a subclass')
 
     def delete_notebook_model(self, name, path=''):
         """Delete notebook by name and path."""
         raise NotImplementedError('must be implemented in a subclass')
 
     def create_notebook_model(self, model=None, path=''):
         """Create a new notebook and return its model with no content."""
         path = path.strip('/')
         if model is None:
             model = {}
         if 'content' not in model:
             metadata = current.new_metadata(name=u'')
             model['content'] = current.new_notebook(metadata=metadata)
         if 'name' not in model:
             model['name'] = self.increment_filename('Untitled', path)
             
         model['path'] = path
         model = self.save_notebook_model(model, model['name'], model['path'])
         return model
 
     def copy_notebook(self, from_name, to_name=None, path=''):
         """Copy an existing notebook and return its new model.
         
         If to_name not specified, increment `from_name-Copy#.ipynb`.
         """
         path = path.strip('/')
         model = self.get_notebook_model(from_name, path)
         if not to_name:
             base = os.path.splitext(from_name)[0] + '-Copy'
             to_name = self.increment_filename(base, path)
         model['name'] = to_name
         model = self.save_notebook_model(model, to_name, path)
         return model
     
     # Checkpoint-related
     
     def create_checkpoint(self, name, path=''):
         """Create a checkpoint of the current state of a notebook
         
         Returns a checkpoint_id for the new checkpoint.
         """
         raise NotImplementedError("must be implemented in a subclass")
     
     def list_checkpoints(self, name, path=''):
         """Return a list of checkpoints for a given notebook"""
         return []
     
     def restore_checkpoint(self, checkpoint_id, name, path=''):
         """Restore a notebook from one of its checkpoints"""
         raise NotImplementedError("must be implemented in a subclass")
 
     def delete_checkpoint(self, checkpoint_id, name, path=''):
         """delete a checkpoint for a notebook"""
         raise NotImplementedError("must be implemented in a subclass")
     
     def log_info(self):
         self.log.info(self.info_string())
     
     def info_string(self):
         return "Serving notebooks"

IPython/parallel/apps/baseapp.py

@@ -1,275 +1,276 b''
 # encoding: utf-8
 """
 The Base Application class for IPython.parallel apps
 
 Authors:
 
 * Brian Granger
 * Min RK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import logging
 import re
 import sys
 
 from subprocess import Popen, PIPE
 
 from IPython.config.application import catch_config_error, LevelFormatter
 from IPython.core import release
 from IPython.core.crashhandler import CrashHandler
 from IPython.core.application import (
     BaseIPythonApplication,
     base_aliases as base_ip_aliases,
     base_flags as base_ip_flags
 )
 from IPython.utils.path import expand_path
+from IPython.utils import py3compat
 from IPython.utils.py3compat import unicode_type
 
 from IPython.utils.traitlets import Unicode, Bool, Instance, Dict
 
 #-----------------------------------------------------------------------------
 # Module errors
 #-----------------------------------------------------------------------------
 
 class PIDFileError(Exception):
     pass
 
 
 #-----------------------------------------------------------------------------
 # Crash handler for this application
 #-----------------------------------------------------------------------------
 
 class ParallelCrashHandler(CrashHandler):
     """sys.excepthook for IPython itself, leaves a detailed report on disk."""
 
     def __init__(self, app):
         contact_name = release.authors['Min'][0]
         contact_email = release.author_email
         bug_tracker = 'https://github.com/ipython/ipython/issues'
         super(ParallelCrashHandler,self).__init__(
             app, contact_name, contact_email, bug_tracker
         )
 
 
 #-----------------------------------------------------------------------------
 # Main application
 #-----------------------------------------------------------------------------
 base_aliases = {}
 base_aliases.update(base_ip_aliases)
 base_aliases.update({
     'work-dir' : 'BaseParallelApplication.work_dir',
     'log-to-file' : 'BaseParallelApplication.log_to_file',
     'clean-logs' : 'BaseParallelApplication.clean_logs',
     'log-url' : 'BaseParallelApplication.log_url',
     'cluster-id' : 'BaseParallelApplication.cluster_id',
 })
 
 base_flags = {
     'log-to-file' : (
         {'BaseParallelApplication' : {'log_to_file' : True}},
         "send log output to a file"
     )
 }
 base_flags.update(base_ip_flags)
 
 class BaseParallelApplication(BaseIPythonApplication):
     """The base Application for IPython.parallel apps
     
     Principle extensions to BaseIPyythonApplication:
     
     * work_dir
     * remote logging via pyzmq
     * IOLoop instance
     """
 
     crash_handler_class = ParallelCrashHandler
     
     def _log_level_default(self):
         # temporarily override default_log_level to INFO
         return logging.INFO
     
     def _log_format_default(self):
         """override default log format to include time"""
         return u"%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s"
 
-    work_dir = Unicode(os.getcwdu(), config=True,
+    work_dir = Unicode(py3compat.getcwd(), config=True,
         help='Set the working dir for the process.'
     )
     def _work_dir_changed(self, name, old, new):
         self.work_dir = unicode_type(expand_path(new))
 
     log_to_file = Bool(config=True,
         help="whether to log to a file")
 
     clean_logs = Bool(False, config=True,
         help="whether to cleanup old logfiles before starting")
 
     log_url = Unicode('', config=True,
         help="The ZMQ URL of the iplogger to aggregate logging.")
 
     cluster_id = Unicode('', config=True,
         help="""String id to add to runtime files, to prevent name collisions when
         using multiple clusters with a single profile simultaneously.
         
         When set, files will be named like: 'ipcontroller-<cluster_id>-engine.json'
         
         Since this is text inserted into filenames, typical recommendations apply:
         Simple character strings are ideal, and spaces are not recommended (but should
         generally work).
         """
     )
     def _cluster_id_changed(self, name, old, new):
         self.name = self.__class__.name
         if new:
             self.name += '-%s'%new
     
     def _config_files_default(self):
         return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
     
     loop = Instance('zmq.eventloop.ioloop.IOLoop')
     def _loop_default(self):
         from zmq.eventloop.ioloop import IOLoop
         return IOLoop.instance()
 
     aliases = Dict(base_aliases)
     flags = Dict(base_flags)
     
     @catch_config_error
     def initialize(self, argv=None):
         """initialize the app"""
         super(BaseParallelApplication, self).initialize(argv)
         self.to_work_dir()
         self.reinit_logging()
         
     def to_work_dir(self):
         wd = self.work_dir
-        if unicode_type(wd) != os.getcwdu():
+        if unicode_type(wd) != py3compat.getcwd():
             os.chdir(wd)
             self.log.info("Changing to working dir: %s" % wd)
         # This is the working dir by now.
         sys.path.insert(0, '')
 
     def reinit_logging(self):
         # Remove old log files
         log_dir = self.profile_dir.log_dir
         if self.clean_logs:
             for f in os.listdir(log_dir):
                 if re.match(r'%s-\d+\.(log|err|out)' % self.name, f):
                     try:
                         os.remove(os.path.join(log_dir, f))
                     except (OSError, IOError):
                         # probably just conflict from sibling process
                         # already removing it
                         pass
         if self.log_to_file:
             # Start logging to the new log file
             log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
             logfile = os.path.join(log_dir, log_filename)
             open_log_file = open(logfile, 'w')
         else:
             open_log_file = None
         if open_log_file is not None:
             while self.log.handlers:
                 self.log.removeHandler(self.log.handlers[0])
             self._log_handler = logging.StreamHandler(open_log_file)
             self.log.addHandler(self._log_handler)
         else:
             self._log_handler = self.log.handlers[0]
         # Add timestamps to log format:
         self._log_formatter = LevelFormatter(self.log_format,
                                                 datefmt=self.log_datefmt)
         self._log_handler.setFormatter(self._log_formatter)
         # do not propagate log messages to root logger
         # ipcluster app will sometimes print duplicate messages during shutdown
         # if this is 1 (default):
         self.log.propagate = False
 
     def write_pid_file(self, overwrite=False):
         """Create a .pid file in the pid_dir with my pid.
 
         This must be called after pre_construct, which sets `self.pid_dir`.
         This raises :exc:`PIDFileError` if the pid file exists already.
         """
         pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             pid = self.get_pid_from_file()
             if not overwrite:
                 raise PIDFileError(
                     'The pid file [%s] already exists. \nThis could mean that this '
                     'server is already running with [pid=%s].' % (pid_file, pid)
                 )
         with open(pid_file, 'w') as f:
             self.log.info("Creating pid file: %s" % pid_file)
             f.write(repr(os.getpid())+'\n')
 
     def remove_pid_file(self):
         """Remove the pid file.
 
         This should be called at shutdown by registering a callback with
         :func:`reactor.addSystemEventTrigger`. This needs to return
         ``None``.
         """
         pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             try:
                 self.log.info("Removing pid file: %s" % pid_file)
                 os.remove(pid_file)
             except:
                 self.log.warn("Error removing the pid file: %s" % pid_file)
 
     def get_pid_from_file(self):
         """Get the pid from the pid file.
 
         If the  pid file doesn't exist a :exc:`PIDFileError` is raised.
         """
         pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
         if os.path.isfile(pid_file):
             with open(pid_file, 'r') as f:
                 s = f.read().strip()
                 try:
                     pid = int(s)
                 except:
                     raise PIDFileError("invalid pid file: %s (contents: %r)"%(pid_file, s))
                 return pid
         else:
             raise PIDFileError('pid file not found: %s' % pid_file)
     
     def check_pid(self, pid):
         if os.name == 'nt':
             try:
                 import ctypes
                 # returns 0 if no such process (of ours) exists
                 # positive int otherwise
                 p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
             except Exception:
                 self.log.warn(
                     "Could not determine whether pid %i is running via `OpenProcess`. "
                     " Making the likely assumption that it is."%pid
                 )
                 return True
             return bool(p)
         else:
             try:
                 p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
                 output,_ = p.communicate()
             except OSError:
                 self.log.warn(
                     "Could not determine whether pid %i is running via `ps x`. "
                     " Making the likely assumption that it is."%pid
                 )
                 return True
             pids = list(map(int, re.findall(r'^\W*\d+', output, re.MULTILINE)))
             return pid in pids

IPython/testing/plugin/ipdoctest.py

@@ -1,764 +1,764 b''
 """Nose Plugin that supports IPython doctests.
 
 Limitations:
 
 - When generating examples for use as doctests, make sure that you have
   pretty-printing OFF.  This can be done either by setting the
   ``PlainTextFormatter.pprint`` option in your configuration file to  False, or
   by interactively disabling it with  %Pprint.  This is required so that IPython
   output matches that of normal Python, which is used by doctest for internal
   execution.
 
 - Do not rely on specific prompt numbers for results (such as using
   '_34==True', for example).  For IPython tests run via an external process the
   prompt numbers may be different, and IPython tests run as normal python code
   won't even have these special _NN variables set at all.
 """
 
 #-----------------------------------------------------------------------------
 # Module imports
 
 # From the standard library
 import doctest
 import inspect
 import logging
 import os
 import re
 import sys
 import traceback
 import unittest
 
 from inspect import getmodule
 
 # We are overriding the default doctest runner, so we need to import a few
 # things from doctest directly
 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
                      _unittest_reportflags, DocTestRunner,
                      _extract_future_flags, pdb, _OutputRedirectingPdb,
                      _exception_traceback,
                      linecache)
 
 # Third-party modules
 import nose.core
 
 from nose.plugins import doctests, Plugin
 from nose.util import anyp, getpackage, test_address, resolve_name, tolist
 
 # Our own imports
-from IPython.utils.py3compat import builtin_mod, PY3
+from IPython.utils.py3compat import builtin_mod, PY3, getcwd
 
 if PY3:
     from io import StringIO
 else:
     from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Module globals and other constants
 #-----------------------------------------------------------------------------
 
 log = logging.getLogger(__name__)
 
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 def is_extension_module(filename):
     """Return whether the given filename is an extension module.
 
     This simply checks that the extension is either .so or .pyd.
     """
     return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
 
 
 class DocTestSkip(object):
     """Object wrapper for doctests to be skipped."""
 
     ds_skip = """Doctest to skip.
     >>> 1 #doctest: +SKIP
     """
 
     def __init__(self,obj):
         self.obj = obj
 
     def __getattribute__(self,key):
         if key == '__doc__':
             return DocTestSkip.ds_skip
         else:
             return getattr(object.__getattribute__(self,'obj'),key)
 
 # Modified version of the one in the stdlib, that fixes a python bug (doctests
 # not found in extension modules, http://bugs.python.org/issue3158)
 class DocTestFinder(doctest.DocTestFinder):
 
     def _from_module(self, module, object):
         """
         Return true if the given object is defined in the given
         module.
         """
         if module is None:
             return True
         elif inspect.isfunction(object):
             return module.__dict__ is object.__globals__
         elif inspect.isbuiltin(object):
             return module.__name__ == object.__module__
         elif inspect.isclass(object):
             return module.__name__ == object.__module__
         elif inspect.ismethod(object):
             # This one may be a bug in cython that fails to correctly set the
             # __module__ attribute of methods, but since the same error is easy
             # to make by extension code writers, having this safety in place
             # isn't such a bad idea
             return module.__name__ == object.__self__.__class__.__module__
         elif inspect.getmodule(object) is not None:
             return module is inspect.getmodule(object)
         elif hasattr(object, '__module__'):
             return module.__name__ == object.__module__
         elif isinstance(object, property):
             return True # [XX] no way not be sure.
         else:
             raise ValueError("object must be a class or function, got %r" % object)
 
     def _find(self, tests, obj, name, module, source_lines, globs, seen):
         """
         Find tests for the given object and any contained objects, and
         add them to `tests`.
         """
         #print '_find for:', obj, name, module  # dbg
         if hasattr(obj,"skip_doctest"):
             #print 'SKIPPING DOCTEST FOR:',obj  # dbg
             obj = DocTestSkip(obj)
 
         doctest.DocTestFinder._find(self,tests, obj, name, module,
                                     source_lines, globs, seen)
 
         # Below we re-run pieces of the above method with manual modifications,
         # because the original code is buggy and fails to correctly identify
         # doctests in extension modules.
 
         # Local shorthands
         from inspect import isroutine, isclass, ismodule
 
         # Look for tests in a module's contained objects.
         if inspect.ismodule(obj) and self._recurse:
             for valname, val in obj.__dict__.items():
                 valname1 = '%s.%s' % (name, valname)
                 if ( (isroutine(val) or isclass(val))
                      and self._from_module(module, val) ):
 
                     self._find(tests, val, valname1, module, source_lines,
                                globs, seen)
 
         # Look for tests in a class's contained objects.
         if inspect.isclass(obj) and self._recurse:
             #print 'RECURSE into class:',obj  # dbg
             for valname, val in obj.__dict__.items():
                 # Special handling for staticmethod/classmethod.
                 if isinstance(val, staticmethod):
                     val = getattr(obj, valname)
                 if isinstance(val, classmethod):
                     val = getattr(obj, valname).__func__
 
                 # Recurse to methods, properties, and nested classes.
                 if ((inspect.isfunction(val) or inspect.isclass(val) or
                      inspect.ismethod(val) or
                       isinstance(val, property)) and
                       self._from_module(module, val)):
                     valname = '%s.%s' % (name, valname)
                     self._find(tests, val, valname, module, source_lines,
                                globs, seen)
 
 
 class IPDoctestOutputChecker(doctest.OutputChecker):
     """Second-chance checker with support for random tests.
 
     If the default comparison doesn't pass, this checker looks in the expected
     output string for flags that tell us to ignore the output.
     """
 
     random_re = re.compile(r'#\s*random\s+')
 
     def check_output(self, want, got, optionflags):
         """Check output, accepting special markers embedded in the output.
 
         If the output didn't pass the default validation but the special string
         '#random' is included, we accept it."""
 
         # Let the original tester verify first, in case people have valid tests
         # that happen to have a comment saying '#random' embedded in.
         ret = doctest.OutputChecker.check_output(self, want, got,
                                                  optionflags)
         if not ret and self.random_re.search(want):
             #print >> sys.stderr, 'RANDOM OK:',want  # dbg
             return True
 
         return ret
 
 
 class DocTestCase(doctests.DocTestCase):
     """Proxy for DocTestCase: provides an address() method that
     returns the correct address for the doctest case. Otherwise
     acts as a proxy to the test case. To provide hints for address(),
     an obj may also be passed -- this will be used as the test object
     for purposes of determining the test address, if it is provided.
     """
 
     # Note: this method was taken from numpy's nosetester module.
 
     # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
     # its constructor that blocks non-default arguments from being passed
     # down into doctest.DocTestCase
 
     def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
                  checker=None, obj=None, result_var='_'):
         self._result_var = result_var
         doctests.DocTestCase.__init__(self, test,
                                       optionflags=optionflags,
                                       setUp=setUp, tearDown=tearDown,
                                       checker=checker)
         # Now we must actually copy the original constructor from the stdlib
         # doctest class, because we can't call it directly and a bug in nose
         # means it never gets passed the right arguments.
 
         self._dt_optionflags = optionflags
         self._dt_checker = checker
         self._dt_test = test
         self._dt_test_globs_ori = test.globs
         self._dt_setUp = setUp
         self._dt_tearDown = tearDown
 
         # XXX - store this runner once in the object!
         runner = IPDocTestRunner(optionflags=optionflags,
                                  checker=checker, verbose=False)
         self._dt_runner = runner
 
 
         # Each doctest should remember the directory it was loaded from, so
         # things like %run work without too many contortions
         self._ori_dir = os.path.dirname(test.filename)
 
     # Modified runTest from the default stdlib
     def runTest(self):
         test = self._dt_test
         runner = self._dt_runner
 
         old = sys.stdout
         new = StringIO()
         optionflags = self._dt_optionflags
 
         if not (optionflags & REPORTING_FLAGS):
             # The option flags don't include any reporting flags,
             # so add the default reporting flags
             optionflags |= _unittest_reportflags
 
         try:
             # Save our current directory and switch out to the one where the
             # test was originally created, in case another doctest did a
             # directory change.  We'll restore this in the finally clause.
-            curdir = os.getcwdu()
+            curdir = getcwd()
             #print 'runTest in dir:', self._ori_dir  # dbg
             os.chdir(self._ori_dir)
 
             runner.DIVIDER = "-"*70
             failures, tries = runner.run(test,out=new.write,
                                          clear_globs=False)
         finally:
             sys.stdout = old
             os.chdir(curdir)
 
         if failures:
             raise self.failureException(self.format_failure(new.getvalue()))
 
     def setUp(self):
         """Modified test setup that syncs with ipython namespace"""
         #print "setUp test", self._dt_test.examples # dbg
         if isinstance(self._dt_test.examples[0], IPExample):
             # for IPython examples *only*, we swap the globals with the ipython
             # namespace, after updating it with the globals (which doctest
             # fills with the necessary info from the module being tested).
             self.user_ns_orig = {}
             self.user_ns_orig.update(_ip.user_ns)
             _ip.user_ns.update(self._dt_test.globs)
             # We must remove the _ key in the namespace, so that Python's
             # doctest code sets it naturally
             _ip.user_ns.pop('_', None)
             _ip.user_ns['__builtins__'] = builtin_mod
             self._dt_test.globs = _ip.user_ns
 
         super(DocTestCase, self).setUp()
 
     def tearDown(self):
 
         # Undo the test.globs reassignment we made, so that the parent class
         # teardown doesn't destroy the ipython namespace
         if isinstance(self._dt_test.examples[0], IPExample):
             self._dt_test.globs = self._dt_test_globs_ori
             _ip.user_ns.clear()
             _ip.user_ns.update(self.user_ns_orig)
 
         # XXX - fperez: I am not sure if this is truly a bug in nose 0.11, but
         # it does look like one to me: its tearDown method tries to run
         #
         # delattr(builtin_mod, self._result_var)
         #
         # without checking that the attribute really is there; it implicitly
         # assumes it should have been set via displayhook.  But if the
         # displayhook was never called, this doesn't necessarily happen.  I
         # haven't been able to find a little self-contained example outside of
         # ipython that would show the problem so I can report it to the nose
         # team, but it does happen a lot in our code.
         #
         # So here, we just protect as narrowly as possible by trapping an
         # attribute error whose message would be the name of self._result_var,
         # and letting any other error propagate.
         try:
             super(DocTestCase, self).tearDown()
         except AttributeError as exc:
             if exc.args[0] != self._result_var:
                 raise
 
 
 # A simple subclassing of the original with a different class name, so we can
 # distinguish and treat differently IPython examples from pure python ones.
 class IPExample(doctest.Example): pass
 
 
 class IPExternalExample(doctest.Example):
     """Doctest examples to be run in an external process."""
 
     def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
                  options=None):
         # Parent constructor
         doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
 
         # An EXTRA newline is needed to prevent pexpect hangs
         self.source += '\n'
 
 
 class IPDocTestParser(doctest.DocTestParser):
     """
     A class used to parse strings containing doctest examples.
 
     Note: This is a version modified to properly recognize IPython input and
     convert any IPython examples into valid Python ones.
     """
     # This regular expression is used to find doctest examples in a
     # string.  It defines three groups: `source` is the source code
     # (including leading indentation and prompts); `indent` is the
     # indentation of the first (PS1) line of the source code; and
     # `want` is the expected output (including leading indentation).
 
     # Classic Python prompts or default IPython ones
     _PS1_PY = r'>>>'
     _PS2_PY = r'\.\.\.'
 
     _PS1_IP = r'In\ \[\d+\]:'
     _PS2_IP = r'\ \ \ \.\.\.+:'
 
     _RE_TPL = r'''
         # Source consists of a PS1 line followed by zero or more PS2 lines.
         (?P<source>
             (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*)    # PS1 line
             (?:\n           [ ]*  (?P<ps2> %s) .*)*)  # PS2 lines
         \n? # a newline
         # Want consists of any non-blank lines that do not start with PS1.
         (?P<want> (?:(?![ ]*$)    # Not a blank line
                      (?![ ]*%s)   # Not a line starting with PS1
                      (?![ ]*%s)   # Not a line starting with PS2
                      .*$\n?       # But any other line
                   )*)
                   '''
 
     _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
                                  re.MULTILINE | re.VERBOSE)
 
     _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
                                  re.MULTILINE | re.VERBOSE)
 
     # Mark a test as being fully random.  In this case, we simply append the
     # random marker ('#random') to each individual example's output.  This way
     # we don't need to modify any other code.
     _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
 
     # Mark tests to be executed in an external process - currently unsupported.
     _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
 
     def ip2py(self,source):
         """Convert input IPython source into valid Python."""
         block = _ip.input_transformer_manager.transform_cell(source)
         if len(block.splitlines()) == 1:
             return _ip.prefilter(block)
         else:
             return block
 
     def parse(self, string, name='<string>'):
         """
         Divide the given string into examples and intervening text,
         and return them as a list of alternating Examples and strings.
         Line numbers for the Examples are 0-based.  The optional
         argument `name` is a name identifying this string, and is only
         used for error messages.
         """
 
         #print 'Parse string:\n',string # dbg
 
         string = string.expandtabs()
         # If all lines begin with the same indentation, then strip it.
         min_indent = self._min_indent(string)
         if min_indent > 0:
             string = '\n'.join([l[min_indent:] for l in string.split('\n')])
 
         output = []
         charno, lineno = 0, 0
 
         # We make 'all random' tests by adding the '# random' mark to every
         # block of output in the test.
         if self._RANDOM_TEST.search(string):
             random_marker = '\n# random'
         else:
             random_marker = ''
 
         # Whether to convert the input from ipython to python syntax
         ip2py = False
         # Find all doctest examples in the string.  First, try them as Python
         # examples, then as IPython ones
         terms = list(self._EXAMPLE_RE_PY.finditer(string))
         if terms:
             # Normal Python example
             #print '-'*70  # dbg
             #print 'PyExample, Source:\n',string  # dbg
             #print '-'*70  # dbg
             Example = doctest.Example
         else:
             # It's an ipython example.  Note that IPExamples are run
             # in-process, so their syntax must be turned into valid python.
             # IPExternalExamples are run out-of-process (via pexpect) so they
             # don't need any filtering (a real ipython will be executing them).
             terms = list(self._EXAMPLE_RE_IP.finditer(string))
             if self._EXTERNAL_IP.search(string):
                 #print '-'*70  # dbg
                 #print 'IPExternalExample, Source:\n',string  # dbg
                 #print '-'*70  # dbg
                 Example = IPExternalExample
             else:
                 #print '-'*70  # dbg
                 #print 'IPExample, Source:\n',string  # dbg
                 #print '-'*70  # dbg
                 Example = IPExample
                 ip2py = True
 
         for m in terms:
             # Add the pre-example text to `output`.
             output.append(string[charno:m.start()])
             # Update lineno (lines before this example)
             lineno += string.count('\n', charno, m.start())
             # Extract info from the regexp match.
             (source, options, want, exc_msg) = \
                      self._parse_example(m, name, lineno,ip2py)
 
             # Append the random-output marker (it defaults to empty in most
             # cases, it's only non-empty for 'all-random' tests):
             want += random_marker
 
             if Example is IPExternalExample:
                 options[doctest.NORMALIZE_WHITESPACE] = True
                 want += '\n'
 
             # Create an Example, and add it to the list.
             if not self._IS_BLANK_OR_COMMENT(source):
                 output.append(Example(source, want, exc_msg,
                                       lineno=lineno,
                                       indent=min_indent+len(m.group('indent')),
                                       options=options))
             # Update lineno (lines inside this example)
             lineno += string.count('\n', m.start(), m.end())
             # Update charno.
             charno = m.end()
         # Add any remaining post-example text to `output`.
         output.append(string[charno:])
         return output
 
     def _parse_example(self, m, name, lineno,ip2py=False):
         """
         Given a regular expression match from `_EXAMPLE_RE` (`m`),
         return a pair `(source, want)`, where `source` is the matched
         example's source code (with prompts and indentation stripped);
         and `want` is the example's expected output (with indentation
         stripped).
 
         `name` is the string's name, and `lineno` is the line number
         where the example starts; both are used for error messages.
 
         Optional:
         `ip2py`: if true, filter the input via IPython to convert the syntax
         into valid python.
         """
 
         # Get the example's indentation level.
         indent = len(m.group('indent'))
 
         # Divide source into lines; check that they're properly
         # indented; and then strip their indentation & prompts.
         source_lines = m.group('source').split('\n')
 
         # We're using variable-length input prompts
         ps1 = m.group('ps1')
         ps2 = m.group('ps2')
         ps1_len = len(ps1)
 
         self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
         if ps2:
             self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
 
         source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
 
         if ip2py:
             # Convert source input from IPython into valid Python syntax
             source = self.ip2py(source)
 
         # Divide want into lines; check that it's properly indented; and
         # then strip the indentation.  Spaces before the last newline should
         # be preserved, so plain rstrip() isn't good enough.
         want = m.group('want')
         want_lines = want.split('\n')
         if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
             del want_lines[-1]  # forget final newline & spaces after it
         self._check_prefix(want_lines, ' '*indent, name,
                            lineno + len(source_lines))
 
         # Remove ipython output prompt that might be present in the first line
         want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
 
         want = '\n'.join([wl[indent:] for wl in want_lines])
 
         # If `want` contains a traceback message, then extract it.
         m = self._EXCEPTION_RE.match(want)
         if m:
             exc_msg = m.group('msg')
         else:
             exc_msg = None
 
         # Extract options from the source.
         options = self._find_options(source, name, lineno)
 
         return source, options, want, exc_msg
 
     def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
         """
         Given the lines of a source string (including prompts and
         leading indentation), check to make sure that every prompt is
         followed by a space character.  If any line is not followed by
         a space character, then raise ValueError.
 
         Note: IPython-modified version which takes the input prompt length as a
         parameter, so that prompts of variable length can be dealt with.
         """
         space_idx = indent+ps1_len
         min_len = space_idx+1
         for i, line in enumerate(lines):
             if len(line) >=  min_len and line[space_idx] != ' ':
                 raise ValueError('line %r of the docstring for %s '
                                  'lacks blank after %s: %r' %
                                  (lineno+i+1, name,
                                   line[indent:space_idx], line))
 
 
 SKIP = doctest.register_optionflag('SKIP')
 
 
 class IPDocTestRunner(doctest.DocTestRunner,object):
     """Test runner that synchronizes the IPython namespace with test globals.
     """
 
     def run(self, test, compileflags=None, out=None, clear_globs=True):
 
         # Hack: ipython needs access to the execution context of the example,
         # so that it can propagate user variables loaded by %run into
         # test.globs.  We put them here into our modified %run as a function
         # attribute.  Our new %run will then only make the namespace update
         # when called (rather than unconconditionally updating test.globs here
         # for all examples, most of which won't be calling %run anyway).
         #_ip._ipdoctest_test_globs = test.globs
         #_ip._ipdoctest_test_filename = test.filename
 
         test.globs.update(_ip.user_ns)
 
         return super(IPDocTestRunner,self).run(test,
                                                compileflags,out,clear_globs)
 
 
 class DocFileCase(doctest.DocFileCase):
     """Overrides to provide filename
     """
     def address(self):
         return (self._dt_test.filename, None, None)
 
 
 class ExtensionDoctest(doctests.Doctest):
     """Nose Plugin that supports doctests in extension modules.
     """
     name = 'extdoctest'   # call nosetests with --with-extdoctest
     enabled = True
 
     def options(self, parser, env=os.environ):
         Plugin.options(self, parser, env)
         parser.add_option('--doctest-tests', action='store_true',
                           dest='doctest_tests',
                           default=env.get('NOSE_DOCTEST_TESTS',True),
                           help="Also look for doctests in test modules. "
                           "Note that classes, methods and functions should "
                           "have either doctests or non-doctest tests, "
                           "not both. [NOSE_DOCTEST_TESTS]")
         parser.add_option('--doctest-extension', action="append",
                           dest="doctestExtension",
                           help="Also look for doctests in files with "
                           "this extension [NOSE_DOCTEST_EXTENSION]")
         # Set the default as a list, if given in env; otherwise
         # an additional value set on the command line will cause
         # an error.
         env_setting = env.get('NOSE_DOCTEST_EXTENSION')
         if env_setting is not None:
             parser.set_defaults(doctestExtension=tolist(env_setting))
 
 
     def configure(self, options, config):
         Plugin.configure(self, options, config)
         # Pull standard doctest plugin out of config; we will do doctesting
         config.plugins.plugins = [p for p in config.plugins.plugins
                                   if p.name != 'doctest']
         self.doctest_tests = options.doctest_tests
         self.extension = tolist(options.doctestExtension)
 
         self.parser = doctest.DocTestParser()
         self.finder = DocTestFinder()
         self.checker = IPDoctestOutputChecker()
         self.globs = None
         self.extraglobs = None
 
 
     def loadTestsFromExtensionModule(self,filename):
         bpath,mod = os.path.split(filename)
         modname = os.path.splitext(mod)[0]
         try:
             sys.path.append(bpath)
             module = __import__(modname)
             tests = list(self.loadTestsFromModule(module))
         finally:
             sys.path.pop()
         return tests
 
     # NOTE: the method below is almost a copy of the original one in nose, with
     # a  few modifications to control output checking.
 
     def loadTestsFromModule(self, module):
         #print '*** ipdoctest - lTM',module  # dbg
 
         if not self.matches(module.__name__):
             log.debug("Doctest doesn't want module %s", module)
             return
 
         tests = self.finder.find(module,globs=self.globs,
                                  extraglobs=self.extraglobs)
         if not tests:
             return
 
         # always use whitespace and ellipsis options
         optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
 
         tests.sort()
         module_file = module.__file__
         if module_file[-4:] in ('.pyc', '.pyo'):
             module_file = module_file[:-1]
         for test in tests:
             if not test.examples:
                 continue
             if not test.filename:
                 test.filename = module_file
 
             yield DocTestCase(test,
                               optionflags=optionflags,
                               checker=self.checker)
 
 
     def loadTestsFromFile(self, filename):
         #print "ipdoctest - from file", filename # dbg
         if is_extension_module(filename):
             for t in self.loadTestsFromExtensionModule(filename):
                 yield t
         else:
             if self.extension and anyp(filename.endswith, self.extension):
                 name = os.path.basename(filename)
                 dh = open(filename)
                 try:
                     doc = dh.read()
                 finally:
                     dh.close()
                 test = self.parser.get_doctest(
                     doc, globs={'__file__': filename}, name=name,
                     filename=filename, lineno=0)
                 if test.examples:
                     #print 'FileCase:',test.examples  # dbg
                     yield DocFileCase(test)
                 else:
                     yield False # no tests to load
 
 
 class IPythonDoctest(ExtensionDoctest):
     """Nose Plugin that supports doctests in extension modules.
     """
     name = 'ipdoctest'   # call nosetests with --with-ipdoctest
     enabled = True
 
     def makeTest(self, obj, parent):
         """Look for doctests in the given object, which will be a
         function, method or class.
         """
         #print 'Plugin analyzing:', obj, parent  # dbg
         # always use whitespace and ellipsis options
         optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
 
         doctests = self.finder.find(obj, module=getmodule(parent))
         if doctests:
             for test in doctests:
                 if len(test.examples) == 0:
                     continue
 
                 yield DocTestCase(test, obj=obj,
                                   optionflags=optionflags,
                                   checker=self.checker)
 
     def options(self, parser, env=os.environ):
         #print "Options for nose plugin:", self.name # dbg
         Plugin.options(self, parser, env)
         parser.add_option('--ipdoctest-tests', action='store_true',
                           dest='ipdoctest_tests',
                           default=env.get('NOSE_IPDOCTEST_TESTS',True),
                           help="Also look for doctests in test modules. "
                           "Note that classes, methods and functions should "
                           "have either doctests or non-doctest tests, "
                           "not both. [NOSE_IPDOCTEST_TESTS]")
         parser.add_option('--ipdoctest-extension', action="append",
                           dest="ipdoctest_extension",
                           help="Also look for doctests in files with "
                           "this extension [NOSE_IPDOCTEST_EXTENSION]")
         # Set the default as a list, if given in env; otherwise
         # an additional value set on the command line will cause
         # an error.
         env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
         if env_setting is not None:
             parser.set_defaults(ipdoctest_extension=tolist(env_setting))
 
     def configure(self, options, config):
         #print "Configuring nose plugin:", self.name # dbg
         Plugin.configure(self, options, config)
         # Pull standard doctest plugin out of config; we will do doctesting
         config.plugins.plugins = [p for p in config.plugins.plugins
                                   if p.name != 'doctest']
         self.doctest_tests = options.ipdoctest_tests
         self.extension = tolist(options.ipdoctest_extension)
 
         self.parser = IPDocTestParser()
         self.finder = DocTestFinder(parser=self.parser)
         self.checker = IPDoctestOutputChecker()
         self.globs = None
         self.extraglobs = None

IPython/utils/_process_win32.py

@@ -1,187 +1,187 b''
 """Windows-specific implementation of process utilities.
 
 This file is only meant to be imported by process.py, not by end-users.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # stdlib
 import os
 import sys
 import ctypes
 
 from ctypes import c_int, POINTER
 from ctypes.wintypes import LPCWSTR, HLOCAL
 from subprocess import STDOUT
 
 # our own imports
 from ._process_common import read_no_interrupt, process_handler, arg_split as py_arg_split
 from . import py3compat
 from .encoding import DEFAULT_ENCODING
 
 #-----------------------------------------------------------------------------
 # Function definitions
 #-----------------------------------------------------------------------------
 
 class AvoidUNCPath(object):
     """A context manager to protect command execution from UNC paths.
 
     In the Win32 API, commands can't be invoked with the cwd being a UNC path.
     This context manager temporarily changes directory to the 'C:' drive on
     entering, and restores the original working directory on exit.
 
     The context manager returns the starting working directory *if* it made a
     change and None otherwise, so that users can apply the necessary adjustment
     to their system calls in the event of a change.
 
     Examples
     --------
     ::
         cmd = 'dir'
         with AvoidUNCPath() as path:
             if path is not None:
                 cmd = '"pushd %s &&"%s' % (path, cmd)
             os.system(cmd)
     """
     def __enter__(self):
-        self.path = os.getcwdu()
+        self.path = py3compat.getcwd()
         self.is_unc_path = self.path.startswith(r"\\")
         if self.is_unc_path:
             # change to c drive (as cmd.exe cannot handle UNC addresses)
             os.chdir("C:")
             return self.path
         else:
             # We return None to signal that there was no change in the working
             # directory
             return None
 
     def __exit__(self, exc_type, exc_value, traceback):
         if self.is_unc_path:
             os.chdir(self.path)
 
 
 def _find_cmd(cmd):
     """Find the full path to a .bat or .exe using the win32api module."""
     try:
         from win32api import SearchPath
     except ImportError:
         raise ImportError('you need to have pywin32 installed for this to work')
     else:
         PATH = os.environ['PATH']
         extensions = ['.exe', '.com', '.bat', '.py']
         path = None
         for ext in extensions:
             try:
                 path = SearchPath(PATH, cmd, ext)[0]
             except:
                 pass
         if path is None:
             raise OSError("command %r not found" % cmd)
         else:
             return path
 
 
 def _system_body(p):
     """Callback for _system."""
     enc = DEFAULT_ENCODING
     for line in read_no_interrupt(p.stdout).splitlines():
         line = line.decode(enc, 'replace')
         print(line, file=sys.stdout)
     for line in read_no_interrupt(p.stderr).splitlines():
         line = line.decode(enc, 'replace')
         print(line, file=sys.stderr)
 
     # Wait to finish for returncode
     return p.wait()
 
 
 def system(cmd):
     """Win32 version of os.system() that works with network shares.
 
     Note that this implementation returns None, as meant for use in IPython.
 
     Parameters
     ----------
     cmd : str
       A command to be executed in the system shell.
 
     Returns
     -------
     None : we explicitly do NOT return the subprocess status code, as this
     utility is meant to be used extensively in IPython, where any return value
     would trigger :func:`sys.displayhook` calls.
     """
     # The controller provides interactivity with both
     # stdin and stdout
     #import _process_win32_controller
     #_process_win32_controller.system(cmd)
 
     with AvoidUNCPath() as path:
         if path is not None:
             cmd = '"pushd %s &&"%s' % (path, cmd)
         return process_handler(cmd, _system_body)
 
 def getoutput(cmd):
     """Return standard output of executing cmd in a shell.
 
     Accepts the same arguments as os.system().
 
     Parameters
     ----------
     cmd : str
       A command to be executed in the system shell.
 
     Returns
     -------
     stdout : str
     """
 
     with AvoidUNCPath() as path:
         if path is not None:
             cmd = '"pushd %s &&"%s' % (path, cmd)
         out = process_handler(cmd, lambda p: p.communicate()[0], STDOUT)
 
     if out is None:
         out = b''
     return py3compat.bytes_to_str(out)
 
 try:
     CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
     CommandLineToArgvW.arg_types = [LPCWSTR, POINTER(c_int)]
     CommandLineToArgvW.restype = POINTER(LPCWSTR)
     LocalFree = ctypes.windll.kernel32.LocalFree
     LocalFree.res_type = HLOCAL
     LocalFree.arg_types = [HLOCAL]
     
     def arg_split(commandline, posix=False, strict=True):
         """Split a command line's arguments in a shell-like manner.
 
         This is a special version for windows that use a ctypes call to CommandLineToArgvW
         to do the argv splitting. The posix paramter is ignored.
         
         If strict=False, process_common.arg_split(...strict=False) is used instead.
         """
         #CommandLineToArgvW returns path to executable if called with empty string.
         if commandline.strip() == "":
             return []
         if not strict:
             # not really a cl-arg, fallback on _process_common
             return py_arg_split(commandline, posix=posix, strict=strict)
         argvn = c_int()
         result_pointer = CommandLineToArgvW(py3compat.cast_unicode(commandline.lstrip()), ctypes.byref(argvn))
         result_array_type = LPCWSTR * argvn.value
         result = [arg for arg in result_array_type.from_address(ctypes.addressof(result_pointer.contents))]
         retval = LocalFree(result_pointer)
         return result
 except AttributeError:
     arg_split = py_arg_split

IPython/utils/_process_win32_controller.py

@@ -1,577 +1,577 b''
 """Windows-specific implementation of process utilities with direct WinAPI.
 
 This file is meant to be used by process.py
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function
 
 # stdlib
 import os, sys, threading
 import ctypes, msvcrt
 
 # local imports
-from .py3compat import unicode_type
+from . import py3compat
 
 # Win32 API types needed for the API calls
 from ctypes import POINTER
 from ctypes.wintypes import HANDLE, HLOCAL, LPVOID, WORD, DWORD, BOOL, \
         ULONG, LPCWSTR
 LPDWORD = POINTER(DWORD)
 LPHANDLE = POINTER(HANDLE)
 ULONG_PTR = POINTER(ULONG)
 class SECURITY_ATTRIBUTES(ctypes.Structure):
     _fields_ = [("nLength", DWORD),
                 ("lpSecurityDescriptor", LPVOID),
                 ("bInheritHandle", BOOL)]
 LPSECURITY_ATTRIBUTES = POINTER(SECURITY_ATTRIBUTES)
 class STARTUPINFO(ctypes.Structure):
     _fields_ = [("cb", DWORD),
                 ("lpReserved", LPCWSTR),
                 ("lpDesktop", LPCWSTR),
                 ("lpTitle", LPCWSTR),
                 ("dwX", DWORD),
                 ("dwY", DWORD),
                 ("dwXSize", DWORD),
                 ("dwYSize", DWORD),
                 ("dwXCountChars", DWORD),
                 ("dwYCountChars", DWORD),
                 ("dwFillAttribute", DWORD),
                 ("dwFlags", DWORD),
                 ("wShowWindow", WORD),
                 ("cbReserved2", WORD),
                 ("lpReserved2", LPVOID),
                 ("hStdInput", HANDLE),
                 ("hStdOutput", HANDLE),
                 ("hStdError", HANDLE)]
 LPSTARTUPINFO = POINTER(STARTUPINFO)
 class PROCESS_INFORMATION(ctypes.Structure):
     _fields_ = [("hProcess", HANDLE),
                 ("hThread", HANDLE),
                 ("dwProcessId", DWORD),
                 ("dwThreadId", DWORD)]
 LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION)
 
 # Win32 API constants needed
 ERROR_HANDLE_EOF = 38
 ERROR_BROKEN_PIPE = 109
 ERROR_NO_DATA = 232
 HANDLE_FLAG_INHERIT = 0x0001
 STARTF_USESTDHANDLES = 0x0100
 CREATE_SUSPENDED = 0x0004
 CREATE_NEW_CONSOLE = 0x0010
 CREATE_NO_WINDOW = 0x08000000
 STILL_ACTIVE = 259
 WAIT_TIMEOUT = 0x0102
 WAIT_FAILED = 0xFFFFFFFF
 INFINITE = 0xFFFFFFFF
 DUPLICATE_SAME_ACCESS = 0x00000002
 ENABLE_ECHO_INPUT = 0x0004
 ENABLE_LINE_INPUT = 0x0002
 ENABLE_PROCESSED_INPUT = 0x0001
 
 # Win32 API functions needed
 GetLastError = ctypes.windll.kernel32.GetLastError
 GetLastError.argtypes = []
 GetLastError.restype = DWORD
 
 CreateFile = ctypes.windll.kernel32.CreateFileW
 CreateFile.argtypes = [LPCWSTR, DWORD, DWORD, LPVOID, DWORD, DWORD, HANDLE]
 CreateFile.restype = HANDLE
 
 CreatePipe = ctypes.windll.kernel32.CreatePipe
 CreatePipe.argtypes = [POINTER(HANDLE), POINTER(HANDLE),
         LPSECURITY_ATTRIBUTES, DWORD]
 CreatePipe.restype = BOOL
 
 CreateProcess = ctypes.windll.kernel32.CreateProcessW
 CreateProcess.argtypes = [LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES,
         LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCWSTR, LPSTARTUPINFO,
         LPPROCESS_INFORMATION]
 CreateProcess.restype = BOOL
 
 GetExitCodeProcess = ctypes.windll.kernel32.GetExitCodeProcess
 GetExitCodeProcess.argtypes = [HANDLE, LPDWORD]
 GetExitCodeProcess.restype = BOOL
 
 GetCurrentProcess = ctypes.windll.kernel32.GetCurrentProcess
 GetCurrentProcess.argtypes = []
 GetCurrentProcess.restype = HANDLE
 
 ResumeThread = ctypes.windll.kernel32.ResumeThread
 ResumeThread.argtypes = [HANDLE]
 ResumeThread.restype = DWORD
 
 ReadFile = ctypes.windll.kernel32.ReadFile
 ReadFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPVOID]
 ReadFile.restype = BOOL
 
 WriteFile = ctypes.windll.kernel32.WriteFile
 WriteFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPVOID]
 WriteFile.restype = BOOL
 
 GetConsoleMode = ctypes.windll.kernel32.GetConsoleMode
 GetConsoleMode.argtypes = [HANDLE, LPDWORD]
 GetConsoleMode.restype = BOOL
 
 SetConsoleMode = ctypes.windll.kernel32.SetConsoleMode
 SetConsoleMode.argtypes = [HANDLE, DWORD]
 SetConsoleMode.restype = BOOL
 
 FlushConsoleInputBuffer = ctypes.windll.kernel32.FlushConsoleInputBuffer
 FlushConsoleInputBuffer.argtypes = [HANDLE]
 FlushConsoleInputBuffer.restype = BOOL
 
 WaitForSingleObject = ctypes.windll.kernel32.WaitForSingleObject
 WaitForSingleObject.argtypes = [HANDLE, DWORD]
 WaitForSingleObject.restype = DWORD
 
 DuplicateHandle = ctypes.windll.kernel32.DuplicateHandle
 DuplicateHandle.argtypes = [HANDLE, HANDLE, HANDLE, LPHANDLE,
         DWORD, BOOL, DWORD]
 DuplicateHandle.restype = BOOL
 
 SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation
 SetHandleInformation.argtypes = [HANDLE, DWORD, DWORD]
 SetHandleInformation.restype = BOOL
 
 CloseHandle = ctypes.windll.kernel32.CloseHandle
 CloseHandle.argtypes = [HANDLE]
 CloseHandle.restype = BOOL
 
 CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
 CommandLineToArgvW.argtypes = [LPCWSTR, POINTER(ctypes.c_int)]
 CommandLineToArgvW.restype = POINTER(LPCWSTR)
 
 LocalFree = ctypes.windll.kernel32.LocalFree
 LocalFree.argtypes = [HLOCAL]
 LocalFree.restype = HLOCAL
 
 class AvoidUNCPath(object):
     """A context manager to protect command execution from UNC paths.
 
     In the Win32 API, commands can't be invoked with the cwd being a UNC path.
     This context manager temporarily changes directory to the 'C:' drive on
     entering, and restores the original working directory on exit.
 
     The context manager returns the starting working directory *if* it made a
     change and None otherwise, so that users can apply the necessary adjustment
     to their system calls in the event of a change.
 
     Examples
     --------
     ::
         cmd = 'dir'
         with AvoidUNCPath() as path:
             if path is not None:
                 cmd = '"pushd %s &&"%s' % (path, cmd)
             os.system(cmd)
     """
     def __enter__(self):
-        self.path = os.getcwdu()
+        self.path = py3compat.getcwd()
         self.is_unc_path = self.path.startswith(r"\\")
         if self.is_unc_path:
             # change to c drive (as cmd.exe cannot handle UNC addresses)
             os.chdir("C:")
             return self.path
         else:
             # We return None to signal that there was no change in the working
             # directory
             return None
 
     def __exit__(self, exc_type, exc_value, traceback):
         if self.is_unc_path:
             os.chdir(self.path)
 
 
 class Win32ShellCommandController(object):
     """Runs a shell command in a 'with' context.
 
     This implementation is Win32-specific.
 
     Example:
         # Runs the command interactively with default console stdin/stdout
         with ShellCommandController('python -i') as scc:
             scc.run()
 
         # Runs the command using the provided functions for stdin/stdout
         def my_stdout_func(s):
             # print or save the string 's'
             write_to_stdout(s)
         def my_stdin_func():
             # If input is available, return it as a string.
             if input_available():
                 return get_input()
             # If no input available, return None after a short delay to
             # keep from blocking.
             else:
                 time.sleep(0.01)
                 return None
       
         with ShellCommandController('python -i') as scc:
             scc.run(my_stdout_func, my_stdin_func)
     """
 
     def __init__(self, cmd, mergeout = True):
         """Initializes the shell command controller.
 
         The cmd is the program to execute, and mergeout is
         whether to blend stdout and stderr into one output
         in stdout. Merging them together in this fashion more
         reliably keeps stdout and stderr in the correct order
         especially for interactive shell usage.
         """
         self.cmd = cmd
         self.mergeout = mergeout
 
     def __enter__(self):
         cmd = self.cmd
         mergeout = self.mergeout
 
         self.hstdout, self.hstdin, self.hstderr = None, None, None
         self.piProcInfo = None
         try:
             p_hstdout, c_hstdout, p_hstderr, \
                     c_hstderr, p_hstdin, c_hstdin = [None]*6
 
             # SECURITY_ATTRIBUTES with inherit handle set to True
             saAttr = SECURITY_ATTRIBUTES()
             saAttr.nLength = ctypes.sizeof(saAttr)
             saAttr.bInheritHandle = True
             saAttr.lpSecurityDescriptor = None
 
             def create_pipe(uninherit):
                 """Creates a Windows pipe, which consists of two handles.
 
                 The 'uninherit' parameter controls which handle is not
                 inherited by the child process.
                 """
                 handles = HANDLE(), HANDLE()
                 if not CreatePipe(ctypes.byref(handles[0]),
                             ctypes.byref(handles[1]), ctypes.byref(saAttr), 0):
                     raise ctypes.WinError()
                 if not SetHandleInformation(handles[uninherit],
                             HANDLE_FLAG_INHERIT, 0):
                     raise ctypes.WinError()
                 return handles[0].value, handles[1].value
 
             p_hstdout, c_hstdout = create_pipe(uninherit=0)
             # 'mergeout' signals that stdout and stderr should be merged.
             # We do that by using one pipe for both of them.
             if mergeout:
                 c_hstderr = HANDLE()
                 if not DuplicateHandle(GetCurrentProcess(), c_hstdout,
                                 GetCurrentProcess(), ctypes.byref(c_hstderr),
                                 0, True, DUPLICATE_SAME_ACCESS):
                     raise ctypes.WinError()
             else:
                 p_hstderr, c_hstderr = create_pipe(uninherit=0)
             c_hstdin,  p_hstdin  = create_pipe(uninherit=1)
 
             # Create the process object
             piProcInfo = PROCESS_INFORMATION()
             siStartInfo = STARTUPINFO()
             siStartInfo.cb = ctypes.sizeof(siStartInfo)
             siStartInfo.hStdInput = c_hstdin
             siStartInfo.hStdOutput = c_hstdout
             siStartInfo.hStdError = c_hstderr
             siStartInfo.dwFlags = STARTF_USESTDHANDLES
             dwCreationFlags = CREATE_SUSPENDED | CREATE_NO_WINDOW # | CREATE_NEW_CONSOLE
 
             if not CreateProcess(None,
                     u"cmd.exe /c " + cmd,
                     None, None, True, dwCreationFlags,
                     None, None, ctypes.byref(siStartInfo),
                     ctypes.byref(piProcInfo)):
                 raise ctypes.WinError()
 
             # Close this process's versions of the child handles
             CloseHandle(c_hstdin)
             c_hstdin = None
             CloseHandle(c_hstdout)
             c_hstdout = None
             if c_hstderr != None:
                 CloseHandle(c_hstderr)
                 c_hstderr = None
 
             # Transfer ownership of the parent handles to the object
             self.hstdin = p_hstdin
             p_hstdin = None
             self.hstdout = p_hstdout
             p_hstdout = None
             if not mergeout:
                 self.hstderr = p_hstderr
                 p_hstderr = None
             self.piProcInfo = piProcInfo
 
         finally:
             if p_hstdin:
                 CloseHandle(p_hstdin)
             if c_hstdin:
                 CloseHandle(c_hstdin)
             if p_hstdout:
                 CloseHandle(p_hstdout)
             if c_hstdout:
                 CloseHandle(c_hstdout)
             if p_hstderr:
                 CloseHandle(p_hstderr)
             if c_hstderr:
                 CloseHandle(c_hstderr)
 
         return self
 
     def _stdin_thread(self, handle, hprocess, func, stdout_func):
         exitCode = DWORD()
         bytesWritten = DWORD(0)
         while True:
             #print("stdin thread loop start")
             # Get the input string (may be bytes or unicode)
             data = func()
 
             # None signals to poll whether the process has exited
             if data is None:
                 #print("checking for process completion")
                 if not GetExitCodeProcess(hprocess, ctypes.byref(exitCode)):
                     raise ctypes.WinError()
                 if exitCode.value != STILL_ACTIVE:
                     return
                 # TESTING: Does zero-sized writefile help?
                 if not WriteFile(handle, "", 0,
                         ctypes.byref(bytesWritten), None):
                     raise ctypes.WinError()
                 continue
             #print("\nGot str %s\n" % repr(data), file=sys.stderr)
 
             # Encode the string to the console encoding
             if isinstance(data, unicode): #FIXME: Python3
                 data = data.encode('utf_8')
 
             # What we have now must be a string of bytes
             if not isinstance(data, str): #FIXME: Python3
                 raise RuntimeError("internal stdin function string error")
 
             # An empty string signals EOF
             if len(data) == 0:
                 return
 
             # In a windows console, sometimes the input is echoed,
             # but sometimes not. How do we determine when to do this?
             stdout_func(data)
             # WriteFile may not accept all the data at once.
             # Loop until everything is processed
             while len(data) != 0:
                 #print("Calling writefile")
                 if not WriteFile(handle, data, len(data),
                         ctypes.byref(bytesWritten), None):
                     # This occurs at exit
                     if GetLastError() == ERROR_NO_DATA:
                         return
                     raise ctypes.WinError()
                 #print("Called writefile")
                 data = data[bytesWritten.value:]
 
     def _stdout_thread(self, handle, func):
         # Allocate the output buffer
         data = ctypes.create_string_buffer(4096)
         while True:
             bytesRead = DWORD(0)
             if not ReadFile(handle, data, 4096,
                         ctypes.byref(bytesRead), None):
                 le = GetLastError()
                 if le == ERROR_BROKEN_PIPE:
                     return
                 else:
                     raise ctypes.WinError()
             # FIXME: Python3
             s = data.value[0:bytesRead.value]
             #print("\nv: %s" % repr(s), file=sys.stderr)
             func(s.decode('utf_8', 'replace'))
 
     def run(self, stdout_func = None, stdin_func = None, stderr_func = None):
         """Runs the process, using the provided functions for I/O.
 
         The function stdin_func should return strings whenever a
         character or characters become available.
         The functions stdout_func and stderr_func are called whenever
         something is printed to stdout or stderr, respectively.
         These functions are called from different threads (but not
         concurrently, because of the GIL).
         """
         if stdout_func == None and stdin_func == None and stderr_func == None:
             return self._run_stdio()
 
         if stderr_func != None and self.mergeout:
             raise RuntimeError("Shell command was initiated with "
                     "merged stdin/stdout, but a separate stderr_func "
                     "was provided to the run() method")
 
         # Create a thread for each input/output handle
         stdin_thread = None
         threads = []
         if stdin_func:
             stdin_thread = threading.Thread(target=self._stdin_thread,
                                 args=(self.hstdin, self.piProcInfo.hProcess,
                                 stdin_func, stdout_func))
         threads.append(threading.Thread(target=self._stdout_thread,
                                     args=(self.hstdout, stdout_func)))
         if not self.mergeout:
             if stderr_func == None:
                 stderr_func = stdout_func
             threads.append(threading.Thread(target=self._stdout_thread,
                                         args=(self.hstderr, stderr_func)))
         # Start the I/O threads and the process
         if ResumeThread(self.piProcInfo.hThread) == 0xFFFFFFFF:
             raise ctypes.WinError()
         if stdin_thread is not None:
             stdin_thread.start()
         for thread in threads:
             thread.start()
         # Wait for the process to complete
         if WaitForSingleObject(self.piProcInfo.hProcess, INFINITE) == \
                     WAIT_FAILED:
             raise ctypes.WinError()
         # Wait for the I/O threads to complete
         for thread in threads:
             thread.join()
 
         # Wait for the stdin thread to complete
         if stdin_thread is not None:
             stdin_thread.join()
 
     def _stdin_raw_nonblock(self):
         """Use the raw Win32 handle of sys.stdin to do non-blocking reads"""
         # WARNING: This is experimental, and produces inconsistent results.
         #          It's possible for the handle not to be appropriate for use
         #          with WaitForSingleObject, among other things.
         handle = msvcrt.get_osfhandle(sys.stdin.fileno())
         result = WaitForSingleObject(handle, 100)
         if result == WAIT_FAILED:
             raise ctypes.WinError()
         elif result == WAIT_TIMEOUT:
             print(".", end='')
             return None
         else:
             data = ctypes.create_string_buffer(256)
             bytesRead = DWORD(0)
             print('?', end='')
 
             if not ReadFile(handle, data, 256,
                         ctypes.byref(bytesRead), None):
                 raise ctypes.WinError()
             # This ensures the non-blocking works with an actual console
             # Not checking the error, so the processing will still work with
             # other handle types
             FlushConsoleInputBuffer(handle)
 
             data = data.value
             data = data.replace('\r\n', '\n')
             data = data.replace('\r', '\n')
             print(repr(data) + " ", end='')
             return data
 
     def _stdin_raw_block(self):
         """Use a blocking stdin read"""
         # The big problem with the blocking read is that it doesn't
         # exit when it's supposed to in all contexts. An extra
         # key-press may be required to trigger the exit.
         try:
             data = sys.stdin.read(1)
             data = data.replace('\r', '\n')
             return data
         except WindowsError as we:
             if we.winerror == ERROR_NO_DATA:
                 # This error occurs when the pipe is closed
                 return None
             else:
                 # Otherwise let the error propagate
                 raise we
 
     def _stdout_raw(self, s):
         """Writes the string to stdout"""
         print(s, end='', file=sys.stdout)
         sys.stdout.flush()
 
     def _stderr_raw(self, s):
         """Writes the string to stdout"""
         print(s, end='', file=sys.stderr)
         sys.stderr.flush()
 
     def _run_stdio(self):
         """Runs the process using the system standard I/O.
 
         IMPORTANT: stdin needs to be asynchronous, so the Python
                    sys.stdin object is not used. Instead,
                    msvcrt.kbhit/getwch are used asynchronously.
         """
         # Disable Line and Echo mode
         #lpMode = DWORD()
         #handle = msvcrt.get_osfhandle(sys.stdin.fileno())
         #if GetConsoleMode(handle, ctypes.byref(lpMode)):
         #    set_console_mode = True
         #    if not SetConsoleMode(handle, lpMode.value &
         #            ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT)):
         #        raise ctypes.WinError()
 
         if self.mergeout:
             return self.run(stdout_func = self._stdout_raw,
                     stdin_func = self._stdin_raw_block)
         else:
             return self.run(stdout_func = self._stdout_raw,
                     stdin_func = self._stdin_raw_block,
                     stderr_func = self._stderr_raw)
 
         # Restore the previous console mode
         #if set_console_mode:
         #    if not SetConsoleMode(handle, lpMode.value):
         #        raise ctypes.WinError()
 
     def __exit__(self, exc_type, exc_value, traceback):
         if self.hstdin:
             CloseHandle(self.hstdin)
             self.hstdin = None
         if self.hstdout:
             CloseHandle(self.hstdout)
             self.hstdout = None
         if self.hstderr:
             CloseHandle(self.hstderr)
             self.hstderr = None
         if self.piProcInfo != None:
             CloseHandle(self.piProcInfo.hProcess)
             CloseHandle(self.piProcInfo.hThread)
             self.piProcInfo = None
 
 
 def system(cmd):
     """Win32 version of os.system() that works with network shares.
 
     Note that this implementation returns None, as meant for use in IPython.
 
     Parameters
     ----------
     cmd : str
       A command to be executed in the system shell.
 
     Returns
     -------
     None : we explicitly do NOT return the subprocess status code, as this
     utility is meant to be used extensively in IPython, where any return value
     would trigger :func:`sys.displayhook` calls.
     """
     with AvoidUNCPath() as path:
         if path is not None:
             cmd = '"pushd %s &&"%s' % (path, cmd)
         with Win32ShellCommandController(cmd) as scc:
             scc.run()
 
 
 if __name__ == "__main__":
     print("Test starting!")
     #system("cmd")
     system("python -i")
     print("Test finished!")

IPython/utils/path.py

@@ -1,574 +1,574 b''
 # encoding: utf-8
 """
 Utilities for path handling.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 import errno
 import shutil
 import random
 import tempfile
 import warnings
 from hashlib import md5
 import glob
 
 import IPython
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.process import system
 from IPython.utils.importstring import import_item
 from IPython.utils import py3compat
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 fs_encoding = sys.getfilesystemencoding()
 
 def _get_long_path_name(path):
     """Dummy no-op."""
     return path
 
 def _writable_dir(path):
     """Whether `path` is a directory, to which the user has write access."""
     return os.path.isdir(path) and os.access(path, os.W_OK)
 
 if sys.platform == 'win32':
     @skip_doctest
     def _get_long_path_name(path):
         """Get a long path name (expand ~) on Windows using ctypes.
 
         Examples
         --------
 
         >>> get_long_path_name('c:\\docume~1')
         u'c:\\\\Documents and Settings'
 
         """
         try:
             import ctypes
         except ImportError:
             raise ImportError('you need to have ctypes installed for this to work')
         _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
         _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
             ctypes.c_uint ]
 
         buf = ctypes.create_unicode_buffer(260)
         rv = _GetLongPathName(path, buf, 260)
         if rv == 0 or rv > 260:
             return path
         else:
             return buf.value
 
 
 def get_long_path_name(path):
     """Expand a path into its long form.
 
     On Windows this expands any ~ in the paths. On other platforms, it is
     a null operation.
     """
     return _get_long_path_name(path)
 
 
 def unquote_filename(name, win32=(sys.platform=='win32')):
     """ On Windows, remove leading and trailing quotes from filenames.
     """
     if win32:
         if name.startswith(("'", '"')) and name.endswith(("'", '"')):
             name = name[1:-1]
     return name
 
 def compress_user(path):
     """Reverse of :func:`os.path.expanduser`
     """
     home = os.path.expanduser('~')
     if path.startswith(home):
         path =  "~" + path[len(home):]
     return path
 
 def get_py_filename(name, force_win32=None):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found.
 
     On Windows, apply Windows semantics to the filename. In particular, remove
     any quoting that has been applied to it. This option can be forced for
     testing purposes.
     """
 
     name = os.path.expanduser(name)
     if force_win32 is None:
         win32 = (sys.platform == 'win32')
     else:
         win32 = force_win32
     name = unquote_filename(name, win32=win32)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError('File `%r` not found.' % name)
 
 
 def filefind(filename, path_dirs=None):
     """Find a file by looking through a sequence of paths.
 
     This iterates through a sequence of paths looking for a file and returns
     the full, absolute path of the first occurence of the file.  If no set of
     path dirs is given, the filename is tested as is, after running through
     :func:`expandvars` and :func:`expanduser`.  Thus a simple call::
 
         filefind('myfile.txt')
 
     will find the file in the current working dir, but::
 
         filefind('~/myfile.txt')
 
     Will find the file in the users home directory.  This function does not
     automatically try any paths, such as the cwd or the user's home directory.
 
     Parameters
     ----------
     filename : str
         The filename to look for.
     path_dirs : str, None or sequence of str
         The sequence of paths to look for the file in.  If None, the filename
         need to be absolute or be in the cwd.  If a string, the string is
         put into a sequence and the searched.  If a sequence, walk through
         each element and join with ``filename``, calling :func:`expandvars`
         and :func:`expanduser` before testing for existence.
 
     Returns
     -------
     Raises :exc:`IOError` or returns absolute path to file.
     """
 
     # If paths are quoted, abspath gets confused, strip them...
     filename = filename.strip('"').strip("'")
     # If the input is an absolute path, just check it exists
     if os.path.isabs(filename) and os.path.isfile(filename):
         return filename
 
     if path_dirs is None:
         path_dirs = ("",)
     elif isinstance(path_dirs, py3compat.string_types):
         path_dirs = (path_dirs,)
 
     for path in path_dirs:
-        if path == '.': path = os.getcwdu()
+        if path == '.': path = py3compat.getcwd()
         testname = expand_path(os.path.join(path, filename))
         if os.path.isfile(testname):
             return os.path.abspath(testname)
 
     raise IOError("File %r does not exist in any of the search paths: %r" %
                   (filename, path_dirs) )
 
 
 class HomeDirError(Exception):
     pass
 
 
 def get_home_dir(require_writable=False):
     """Return the 'home' directory, as a unicode string.
 
     Uses os.path.expanduser('~'), and checks for writability.
     
     See stdlib docs for how this is determined.
     $HOME is first priority on *ALL* platforms.
     
     Parameters
     ----------
     
     require_writable : bool [default: False]
         if True:
             guarantees the return value is a writable directory, otherwise
             raises HomeDirError
         if False:
             The path is resolved, but it is not guaranteed to exist or be writable.
     """
 
     homedir = os.path.expanduser('~')
     # Next line will make things work even when /home/ is a symlink to
     # /usr/home as it is on FreeBSD, for example
     homedir = os.path.realpath(homedir)
     
     if not _writable_dir(homedir) and os.name == 'nt':
         # expanduser failed, use the registry to get the 'My Documents' folder.
         try:
             try:
                 import winreg as wreg  # Py 3
             except ImportError:
                 import _winreg as wreg  # Py 2
             key = wreg.OpenKey(
                 wreg.HKEY_CURRENT_USER,
                 "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
             )
             homedir = wreg.QueryValueEx(key,'Personal')[0]
             key.Close()
         except:
             pass
     
     if (not require_writable) or _writable_dir(homedir):
         return py3compat.cast_unicode(homedir, fs_encoding)
     else:
         raise HomeDirError('%s is not a writable dir, '
                 'set $HOME environment variable to override' % homedir)
 
 def get_xdg_dir():
     """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
 
     This is only for non-OS X posix (Linux,Unix,etc.) systems.
     """
 
     env = os.environ
 
     if os.name == 'posix' and sys.platform != 'darwin':
         # Linux, Unix, AIX, etc.
         # use ~/.config if empty OR not set
         xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
         if xdg and _writable_dir(xdg):
             return py3compat.cast_unicode(xdg, fs_encoding)
 
     return None
 
 
 def get_xdg_cache_dir():
     """Return the XDG_CACHE_HOME, if it is defined and exists, else None.
 
     This is only for non-OS X posix (Linux,Unix,etc.) systems.
     """
 
     env = os.environ
 
     if os.name == 'posix' and sys.platform != 'darwin':
         # Linux, Unix, AIX, etc.
         # use ~/.cache if empty OR not set
         xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
         if xdg and _writable_dir(xdg):
             return py3compat.cast_unicode(xdg, fs_encoding)
 
     return None
 
 
 def get_ipython_dir():
     """Get the IPython directory for this platform and user.
 
     This uses the logic in `get_home_dir` to find the home directory
     and then adds .ipython to the end of the path.
     """
 
     env = os.environ
     pjoin = os.path.join
 
 
     ipdir_def = '.ipython'
     xdg_def = 'ipython'
 
     home_dir = get_home_dir()
     xdg_dir = get_xdg_dir()
     
     # import pdb; pdb.set_trace()  # dbg
     if 'IPYTHON_DIR' in env:
         warnings.warn('The environment variable IPYTHON_DIR is deprecated. '
                       'Please use IPYTHONDIR instead.')
     ipdir = env.get('IPYTHONDIR', env.get('IPYTHON_DIR', None))
     if ipdir is None:
         # not set explicitly, use XDG_CONFIG_HOME or HOME
         home_ipdir = pjoin(home_dir, ipdir_def)
         if xdg_dir:
             # use XDG, as long as the user isn't already
             # using $HOME/.ipython and *not* XDG/ipython
 
             xdg_ipdir = pjoin(xdg_dir, xdg_def)
 
             if _writable_dir(xdg_ipdir) or not _writable_dir(home_ipdir):
                 ipdir = xdg_ipdir
 
         if ipdir is None:
             # not using XDG
             ipdir = home_ipdir
 
     ipdir = os.path.normpath(os.path.expanduser(ipdir))
 
     if os.path.exists(ipdir) and not _writable_dir(ipdir):
         # ipdir exists, but is not writable
         warnings.warn("IPython dir '%s' is not a writable location,"
                         " using a temp directory."%ipdir)
         ipdir = tempfile.mkdtemp()
     elif not os.path.exists(ipdir):
         parent = os.path.dirname(ipdir)
         if not _writable_dir(parent):
             # ipdir does not exist and parent isn't writable
             warnings.warn("IPython parent '%s' is not a writable location,"
                         " using a temp directory."%parent)
             ipdir = tempfile.mkdtemp()
 
     return py3compat.cast_unicode(ipdir, fs_encoding)
 
 
 def get_ipython_cache_dir():
     """Get the cache directory it is created if it does not exist."""
     xdgdir = get_xdg_cache_dir()
     if xdgdir is None:
         return get_ipython_dir()
     ipdir = os.path.join(xdgdir, "ipython")
     if not os.path.exists(ipdir) and _writable_dir(xdgdir):
         os.makedirs(ipdir)
     elif not _writable_dir(xdgdir):
         return get_ipython_dir()
 
     return py3compat.cast_unicode(ipdir, fs_encoding)
 
 
 def get_ipython_package_dir():
     """Get the base directory where IPython itself is installed."""
     ipdir = os.path.dirname(IPython.__file__)
     return py3compat.cast_unicode(ipdir, fs_encoding)
 
 
 def get_ipython_module_path(module_str):
     """Find the path to an IPython module in this version of IPython.
 
     This will always find the version of the module that is in this importable
     IPython package. This will always return the path to the ``.py``
     version of the module.
     """
     if module_str == 'IPython':
         return os.path.join(get_ipython_package_dir(), '__init__.py')
     mod = import_item(module_str)
     the_path = mod.__file__.replace('.pyc', '.py')
     the_path = the_path.replace('.pyo', '.py')
     return py3compat.cast_unicode(the_path, fs_encoding)
 
 def locate_profile(profile='default'):
     """Find the path to the folder associated with a given profile.
     
     I.e. find $IPYTHONDIR/profile_whatever.
     """
     from IPython.core.profiledir import ProfileDir, ProfileDirError
     try:
         pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)
     except ProfileDirError:
         # IOError makes more sense when people are expecting a path
         raise IOError("Couldn't find profile %r" % profile)
     return pd.location
 
 def expand_path(s):
     """Expand $VARS and ~names in a string, like a shell
 
     :Examples:
 
        In [2]: os.environ['FOO']='test'
 
        In [3]: expand_path('variable FOO is $FOO')
        Out[3]: 'variable FOO is test'
     """
     # This is a pretty subtle hack. When expand user is given a UNC path
     # on Windows (\\server\share$\%username%), os.path.expandvars, removes
     # the $ to get (\\server\share\%username%). I think it considered $
     # alone an empty var. But, we need the $ to remains there (it indicates
     # a hidden share).
     if os.name=='nt':
         s = s.replace('$\\', 'IPYTHON_TEMP')
     s = os.path.expandvars(os.path.expanduser(s))
     if os.name=='nt':
         s = s.replace('IPYTHON_TEMP', '$\\')
     return s
 
 
 def unescape_glob(string):
     """Unescape glob pattern in `string`."""
     def unescape(s):
         for pattern in '*[]!?':
             s = s.replace(r'\{0}'.format(pattern), pattern)
         return s
     return '\\'.join(map(unescape, string.split('\\\\')))
 
 
 def shellglob(args):
     """
     Do glob expansion for each element in `args` and return a flattened list.
 
     Unmatched glob pattern will remain as-is in the returned list.
 
     """
     expanded = []
     # Do not unescape backslash in Windows as it is interpreted as
     # path separator:
     unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
     for a in args:
         expanded.extend(glob.glob(a) or [unescape(a)])
     return expanded
 
 
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         system(cmd)
 
 def filehash(path):
     """Make an MD5 hash of a file, ignoring any differences in line
     ending characters."""
     with open(path, "rU") as f:
         return md5(py3compat.str_to_bytes(f.read())).hexdigest()
 
 # If the config is unmodified from the default, we'll just delete it.
 # These are consistent for 0.10.x, thankfully. We're not going to worry about
 # older versions.
 old_config_md5 = {'ipy_user_conf.py': 'fc108bedff4b9a00f91fa0a5999140d3',
                   'ipythonrc': '12a68954f3403eea2eec09dc8fe5a9b5'}
 
 def check_for_old_config(ipython_dir=None):
     """Check for old config files, and present a warning if they exist.
 
     A link to the docs of the new config is included in the message.
 
     This should mitigate confusion with the transition to the new
     config system in 0.11.
     """
     if ipython_dir is None:
         ipython_dir = get_ipython_dir()
 
     old_configs = ['ipy_user_conf.py', 'ipythonrc', 'ipython_config.py']
     warned = False
     for cfg in old_configs:
         f = os.path.join(ipython_dir, cfg)
         if os.path.exists(f):
             if filehash(f) == old_config_md5.get(cfg, ''):
                 os.unlink(f)
             else:
                 warnings.warn("Found old IPython config file %r (modified by user)"%f)
                 warned = True
 
     if warned:
         warnings.warn("""
   The IPython configuration system has changed as of 0.11, and these files will
   be ignored. See http://ipython.github.com/ipython-doc/dev/config for details
   of the new config system.
   To start configuring IPython, do `ipython profile create`, and edit
   `ipython_config.py` in <ipython_dir>/profile_default.
   If you need to leave the old config files in place for an older version of
   IPython and want to suppress this warning message, set
   `c.InteractiveShellApp.ignore_old_config=True` in the new config.""")
 
 def get_security_file(filename, profile='default'):
     """Return the absolute path of a security file given by filename and profile
     
     This allows users and developers to find security files without
     knowledge of the IPython directory structure. The search path
     will be ['.', profile.security_dir]
     
     Parameters
     ----------
     
     filename : str
         The file to be found. If it is passed as an absolute path, it will
         simply be returned.
     profile : str [default: 'default']
         The name of the profile to search.  Leaving this unspecified
         The file to be found. If it is passed as an absolute path, fname will
         simply be returned.
     
     Returns
     -------
     Raises :exc:`IOError` if file not found or returns absolute path to file.
     """
     # import here, because profiledir also imports from utils.path
     from IPython.core.profiledir import ProfileDir
     try:
         pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)
     except Exception:
         # will raise ProfileDirError if no such profile
         raise IOError("Profile %r not found")
     return filefind(filename, ['.', pd.security_dir])
 
 
 ENOLINK = 1998
 
 def link(src, dst):
     """Hard links ``src`` to ``dst``, returning 0 or errno.
 
     Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
     supported by the operating system.
     """
 
     if not hasattr(os, "link"):
         return ENOLINK
     link_errno = 0
     try:
         os.link(src, dst)
     except OSError as e:
         link_errno = e.errno
     return link_errno
 
 
 def link_or_copy(src, dst):
     """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.
 
     Attempts to maintain the semantics of ``shutil.copy``.
 
     Because ``os.link`` does not overwrite files, a unique temporary file
     will be used if the target already exists, then that file will be moved
     into place.
     """
 
     if os.path.isdir(dst):
         dst = os.path.join(dst, os.path.basename(src))
 
     link_errno = link(src, dst)
     if link_errno == errno.EEXIST:
         new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
         try:
             link_or_copy(src, new_dst)
         except:
             try:
                 os.remove(new_dst)
             except OSError:
                 pass
             raise
         os.rename(new_dst, dst)
     elif link_errno != 0:
         # Either link isn't supported, or the filesystem doesn't support
         # linking, or 'src' and 'dst' are on different filesystems.
         shutil.copy(src, dst)

IPython/utils/process.py

@@ -1,122 +1,123 b''
 # encoding: utf-8
 """
 Utilities for working with external processes.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib
 import os
 import sys
 import shlex
 
 # Our own
 if sys.platform == 'win32':
     from ._process_win32 import _find_cmd, system, getoutput, AvoidUNCPath, arg_split
 else:
     from ._process_posix import _find_cmd, system, getoutput, arg_split
 
 
 from ._process_common import getoutputerror, get_output_error_code
+from . import py3compat
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 class FindCmdError(Exception):
     pass
 
 
 def find_cmd(cmd):
     """Find absolute path to executable cmd in a cross platform manner.
 
     This function tries to determine the full path to a command line program
     using `which` on Unix/Linux/OS X and `win32api` on Windows.  Most of the
     time it will use the version that is first on the users `PATH`.
 
     Warning, don't use this to find IPython command line programs as there
     is a risk you will find the wrong one.  Instead find those using the
     following code and looking for the application itself::
 
         from IPython.utils.path import get_ipython_module_path
         from IPython.utils.process import pycmd2argv
         argv = pycmd2argv(get_ipython_module_path('IPython.terminal.ipapp'))
 
     Parameters
     ----------
     cmd : str
         The command line program to look for.
     """
     try:
         path = _find_cmd(cmd).rstrip()
     except OSError:
         raise FindCmdError('command could not be found: %s' % cmd)
     # which returns empty if not found
     if path == '':
         raise FindCmdError('command could not be found: %s' % cmd)
     return os.path.abspath(path)
 
 
 def is_cmd_found(cmd):
     """Check whether executable `cmd` exists or not and return a bool."""
     try:
         find_cmd(cmd)
         return True
     except FindCmdError:
         return False
 
 
 def pycmd2argv(cmd):
     r"""Take the path of a python command and return a list (argv-style).
 
     This only works on Python based command line programs and will find the
     location of the ``python`` executable using ``sys.executable`` to make
     sure the right version is used.
 
     For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
     .com or .bat, and [, cmd] otherwise.
 
     Parameters
     ----------
     cmd : string
       The path of the command.
 
     Returns
     -------
     argv-style list.
     """
     ext = os.path.splitext(cmd)[1]
     if ext in ['.exe', '.com', '.bat']:
         return [cmd]
     else:
         return [sys.executable, cmd]
 
 
 def abbrev_cwd():
     """ Return abbreviated version of cwd, e.g. d:mydir """
-    cwd = os.getcwdu().replace('\\','/')
+    cwd = py3compat.getcwd().replace('\\','/')
     drivepart = ''
     tail = cwd
     if sys.platform == 'win32':
         if len(cwd) < 4:
             return cwd
         drivepart,tail = os.path.splitdrive(cwd)
 
 
     parts = tail.split('/')
     if len(parts) > 2:
         tail = '/'.join(parts[-2:])
 
     return (drivepart + (
         cwd == '/' and '/' or tail))

IPython/utils/py3compat.py

@@ -1,239 +1,242 b''
 # coding: utf-8
 """Compatibility tricks for Python 3. Mainly to do with unicode."""
 import functools
+import os
 import sys
 import re
 import types
 
 from .encoding import DEFAULT_ENCODING
 
 orig_open = open
 
 def no_code(x, encoding=None):
     return x
 
 def decode(s, encoding=None):
     encoding = encoding or DEFAULT_ENCODING
     return s.decode(encoding, "replace")
 
 def encode(u, encoding=None):
     encoding = encoding or DEFAULT_ENCODING
     return u.encode(encoding, "replace")
 
 
 def cast_unicode(s, encoding=None):
     if isinstance(s, bytes):
         return decode(s, encoding)
     return s
 
 def cast_bytes(s, encoding=None):
     if not isinstance(s, bytes):
         return encode(s, encoding)
     return s
 
 def _modify_str_or_docstring(str_change_func):
     @functools.wraps(str_change_func)
     def wrapper(func_or_str):
         if isinstance(func_or_str, string_types):
             func = None
             doc = func_or_str
         else:
             func = func_or_str
             doc = func.__doc__
         
         doc = str_change_func(doc)
         
         if func:
             func.__doc__ = doc
             return func
         return doc
     return wrapper
 
 def safe_unicode(e):
     """unicode(e) with various fallbacks. Used for exceptions, which may not be
     safe to call unicode() on.
     """
     try:
         return unicode_type(e)
     except UnicodeError:
         pass
 
     try:
         return str_to_unicode(str(e))
     except UnicodeError:
         pass
 
     try:
         return str_to_unicode(repr(e))
     except UnicodeError:
         pass
 
     return u'Unrecoverably corrupt evalue'
 
 if sys.version_info[0] >= 3:
     PY3 = True
     
     input = input
     builtin_mod_name = "builtins"
     import builtins as builtin_mod
     
     str_to_unicode = no_code
     unicode_to_str = no_code
     str_to_bytes = encode
     bytes_to_str = decode
     cast_bytes_py2 = no_code
     
     string_types = (str,)
     unicode_type = str
     
     def isidentifier(s, dotted=False):
         if dotted:
             return all(isidentifier(a) for a in s.split("."))
         return s.isidentifier()
     
     open = orig_open
     xrange = range
     def iteritems(d): return iter(d.items())
     def itervalues(d): return iter(d.values())
+    getcwd = os.getcwd
     
     MethodType = types.MethodType
     
     def execfile(fname, glob, loc=None):
         loc = loc if (loc is not None) else glob
         with open(fname, 'rb') as f:
             exec(compile(f.read(), fname, 'exec'), glob, loc)
     
     # Refactor print statements in doctests.
     _print_statement_re = re.compile(r"\bprint (?P<expr>.*)$", re.MULTILINE)
     def _print_statement_sub(match):
         expr = match.groups('expr')
         return "print(%s)" % expr
     
     @_modify_str_or_docstring
     def doctest_refactor_print(doc):
         """Refactor 'print x' statements in a doctest to print(x) style. 2to3
         unfortunately doesn't pick up on our doctests.
         
         Can accept a string or a function, so it can be used as a decorator."""
         return _print_statement_re.sub(_print_statement_sub, doc)
     
     # Abstract u'abc' syntax:
     @_modify_str_or_docstring
     def u_format(s):
         """"{u}'abc'" --> "'abc'" (Python 3)
         
         Accepts a string or a function, so it can be used as a decorator."""
         return s.format(u='')
 
 else:
     PY3 = False
     
     input = raw_input
     builtin_mod_name = "__builtin__"
     import __builtin__ as builtin_mod
     
     str_to_unicode = decode
     unicode_to_str = encode
     str_to_bytes = no_code
     bytes_to_str = no_code
     cast_bytes_py2 = cast_bytes
     
     string_types = (str, unicode)
     unicode_type = unicode
     
     import re
     _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")
     def isidentifier(s, dotted=False):
         if dotted:
             return all(isidentifier(a) for a in s.split("."))
         return bool(_name_re.match(s))
     
     class open(object):
         """Wrapper providing key part of Python 3 open() interface."""
         def __init__(self, fname, mode="r", encoding="utf-8"):
             self.f = orig_open(fname, mode)
             self.enc = encoding
         
         def write(self, s):
             return self.f.write(s.encode(self.enc))
         
         def read(self, size=-1):
             return self.f.read(size).decode(self.enc)
         
         def close(self):
             return self.f.close()
         
         def __enter__(self):
             return self
         
         def __exit__(self, etype, value, traceback):
             self.f.close()
     
     xrange = xrange
     def iteritems(d): return d.iteritems()
     def itervalues(d): return d.itervalues()
+    getcwd = os.getcwdu
 
     def MethodType(func, instance):
         return types.MethodType(func, instance, type(instance))
     
     # don't override system execfile on 2.x:
     execfile = execfile
     
     def doctest_refactor_print(func_or_str):
         return func_or_str
 
 
     # Abstract u'abc' syntax:
     @_modify_str_or_docstring
     def u_format(s):
         """"{u}'abc'" --> "u'abc'" (Python 2)
         
         Accepts a string or a function, so it can be used as a decorator."""
         return s.format(u='u')
 
     if sys.platform == 'win32':
         def execfile(fname, glob=None, loc=None):
             loc = loc if (loc is not None) else glob
             # The rstrip() is necessary b/c trailing whitespace in files will
             # cause an IndentationError in Python 2.6 (this was fixed in 2.7,
             # but we still support 2.6).  See issue 1027.
             scripttext = builtin_mod.open(fname).read().rstrip() + '\n'
             # compile converts unicode filename to str assuming
             # ascii. Let's do the conversion before calling compile
             if isinstance(fname, unicode):
                 filename = unicode_to_str(fname)
             else:
                 filename = fname
             exec(compile(scripttext, filename, 'exec'), glob, loc)
     else:
         def execfile(fname, *where):
             if isinstance(fname, unicode):
                 filename = fname.encode(sys.getfilesystemencoding())
             else:
                 filename = fname
             builtin_mod.execfile(filename, *where)
 
 # Parts below taken from six:
 # Copyright (c) 2010-2013 Benjamin Peterson
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in all
 # copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
 def with_metaclass(meta, *bases):
     """Create a base class with a metaclass."""
     return meta("NewBase", bases, {})

IPython/utils/terminal.py

@@ -1,162 +1,164 b''
 # encoding: utf-8
 """
 Utilities for working with terminals.
 
 Authors:
 
 * Brian E. Granger
 * Fernando Perez
 * Alexander Belchenko (e-mail: bialix AT ukr.net)
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import struct
 import sys
 import warnings
 
+from . import py3compat
+
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 # This variable is part of the expected API of the module:
 ignore_termtitle = True
 
 
 def _term_clear():
     pass
 
 
 if os.name == 'posix':
     def _term_clear():
         os.system('clear')
 
 
 if sys.platform == 'win32':
     def _term_clear():
         os.system('cls')
 
 
 def term_clear():
     _term_clear()
 
 
 def toggle_set_term_title(val):
     """Control whether set_term_title is active or not.
 
     set_term_title() allows writing to the console titlebar.  In embedded
     widgets this can cause problems, so this call can be used to toggle it on
     or off as needed.
 
     The default state of the module is for the function to be disabled.
 
     Parameters
     ----------
       val : bool
         If True, set_term_title() actually writes to the terminal (using the
         appropriate platform-specific module).  If False, it is a no-op.
     """
     global ignore_termtitle
     ignore_termtitle = not(val)
 
 
 def _set_term_title(*args,**kw):
     """Dummy no-op."""
     pass
 
 
 def _set_term_title_xterm(title):
     """ Change virtual terminal title in xterm-workalikes """
     sys.stdout.write('\033]0;%s\007' % title)
 
 if os.name == 'posix':
     TERM = os.environ.get('TERM','')
     if TERM.startswith('xterm'):
         _set_term_title = _set_term_title_xterm
 
 
 if sys.platform == 'win32':
     try:
         import ctypes
 
         SetConsoleTitleW = ctypes.windll.kernel32.SetConsoleTitleW
         SetConsoleTitleW.argtypes = [ctypes.c_wchar_p]
     
         def _set_term_title(title):
             """Set terminal title using ctypes to access the Win32 APIs."""
             SetConsoleTitleW(title)
     except ImportError:
         def _set_term_title(title):
             """Set terminal title using the 'title' command."""
             global ignore_termtitle
 
             try:
                 # Cannot be on network share when issuing system commands
-                curr = os.getcwdu()
+                curr = py3compat.getcwd()
                 os.chdir("C:")
                 ret = os.system("title " + title)
             finally:
                 os.chdir(curr)
             if ret:
                 # non-zero return code signals error, don't try again
                 ignore_termtitle = True
 
 
 def set_term_title(title):
     """Set terminal title using the necessary platform-dependent calls."""
     if ignore_termtitle:
         return
     _set_term_title(title)
 
 
 def freeze_term_title():
     warnings.warn("This function is deprecated, use toggle_set_term_title()")
     global ignore_termtitle
     ignore_termtitle = True
 
 
 def get_terminal_size(defaultx=80, defaulty=25):
     return defaultx, defaulty
 
 
 if sys.platform == 'win32':
     def get_terminal_size(defaultx=80, defaulty=25):
         """Return size of current terminal console.
 
         This function try to determine actual size of current working
         console window and return tuple (sizex, sizey) if success,
         or default size (defaultx, defaulty) otherwise.
 
         Dependencies: ctypes should be installed.
 
         Author: Alexander Belchenko (e-mail: bialix AT ukr.net)
         """
         try:
             import ctypes
         except ImportError:
             return defaultx, defaulty
 
         h = ctypes.windll.kernel32.GetStdHandle(-11)
         csbi = ctypes.create_string_buffer(22)
         res = ctypes.windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
 
         if res:
             (bufx, bufy, curx, cury, wattr,
              left, top, right, bottom, maxx, maxy) = struct.unpack(
                 "hhhhHhhhhhh", csbi.raw)
             sizex = right - left + 1
             sizey = bottom - top + 1
             return (sizex, sizey)
         else:
             return (defaultx, defaulty)