Start using py3compat module.

IPython/config/loader.py

@@ -18,13 +18,13 b' Authors'
 # Imports
 #-----------------------------------------------------------------------------
 
-import __builtin__
+import __builtin__ as builtin_mod
 import re
 import sys
 
 from IPython.external import argparse
 from IPython.utils.path import filefind, get_ipython_dir
-from IPython.utils import warn
+from IPython.utils import py3compat, warn
 
 #-----------------------------------------------------------------------------
 # Exceptions
@@ -131,7 +131,7 b' class Config(dict):'
         # that you can't have section or attribute names that are 
         # builtins.
         try:
-            return getattr(__builtin__, key)
+            return getattr(builtin_mod, key)
         except AttributeError:
             pass
         if is_section_key(key):
@@ -146,7 +146,7 b' class Config(dict):'
 
     def __setitem__(self, key, value):
         # Don't allow names in __builtin__ to be modified.
-        if hasattr(__builtin__, key):
+        if hasattr(builtin_mod, key):
             raise ConfigError('Config variable names cannot have the same name '
                               'as a Python builtin: %s' % key)
         if self._is_section_key(key):
@@ -312,7 +312,7 b' class PyFileConfigLoader(FileConfigLoader):'
         namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
         fs_encoding = sys.getfilesystemencoding() or 'ascii'
         conf_filename = self.full_filename.encode(fs_encoding)
-        execfile(conf_filename, namespace)
+        py3compat.execfile(conf_filename, namespace)
 
     def _convert_to_config(self):
         if self.data is None:
@@ -586,13 +586,7 b' class ArgParseConfigLoader(CommandLineConfigLoader):'
     def _parse_args(self, args):
         """self.parser->self.parsed_data"""
         # decode sys.argv to support unicode command-line options
-        uargs = []
-        for a in args:
-            if isinstance(a, str):
-                # don't decode if we already got unicode
-                a = a.decode(sys.stdin.encoding or 
-                                            sys.getdefaultencoding())
-            uargs.append(a)
+        uargs = [py3compat.cast_unicode(a) for a in args]
         self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
 
     def _convert_to_config(self):

IPython/core/completerlib.py

@@ -30,6 +30,7 b' from zipimport import zipimporter'
 # Our own imports
 from IPython.core.completer import expand_user, compress_user
 from IPython.core.error import TryNext
+from IPython.utils import py3compat
 
 # FIXME: this should be pulled in with the right call via the component system
 from IPython.core.ipapi import get as get_ipython
@@ -66,10 +67,9 b' def shlex_split(x):'
     # Example:
     # %run "c:/python -> ['%run','"c:/python']
 
-    # shlex.split has unicode bugs, so encode first to str
-    if isinstance(x, unicode):
-        # don't raise errors on encoding:
-        x = x.encode(sys.stdin.encoding or sys.getdefaultencoding(), 'replace')
+    # shlex.split has unicode bugs in Python 2, so encode first to str
+    if not py3compat.PY3:
+        x = py3compat.cast_bytes(x)
 
     endofline = []
     while x != '':

IPython/core/interactiveshell.py

@@ -17,7 +17,7 b''
 from __future__ import with_statement
 from __future__ import absolute_import
 
-import __builtin__
+import __builtin__ as builtin_mod
 import __future__
 import abc
 import ast
@@ -61,6 +61,7 b' from IPython.core.profiledir import ProfileDir'
 from IPython.external.Itpl import ItplNS
 from IPython.utils import PyColorize
 from IPython.utils import io
+from IPython.utils import py3compat
 from IPython.utils.doctestreload import doctest_reload
 from IPython.utils.io import ask_yes_no, rprint
 from IPython.utils.ipstruct import Struct
@@ -979,13 +980,13 b' class InteractiveShell(SingletonConfigurable, Magic):'
             # Set __name__ to __main__ to better match the behavior of the
             # normal interpreter.
             user_ns = {'__name__'     :'__main__',
-                       '__builtin__' : __builtin__,
-                       '__builtins__' : __builtin__,
+                       py3compat.builtin_mod_name: builtin_mod,
+                       '__builtins__' : builtin_mod,
                       }
         else:
             user_ns.setdefault('__name__','__main__')
-            user_ns.setdefault('__builtin__',__builtin__)
-            user_ns.setdefault('__builtins__',__builtin__)
+            user_ns.setdefault(py3compat.builtin_mod_name,builtin_mod)
+            user_ns.setdefault('__builtins__',builtin_mod)
 
         if user_global_ns is None:
             user_global_ns = user_ns
@@ -1255,14 +1256,15 b' class InteractiveShell(SingletonConfigurable, Magic):'
 
         Has special code to detect magic functions.
         """
-        #oname = oname.strip()
+        oname = oname.strip()
         #print '1- oname: <%r>' % oname  # dbg
-        try:
-            oname = oname.strip().encode('ascii')
-            #print '2- oname: <%r>' % oname  # dbg
-        except UnicodeEncodeError:
-            print 'Python identifiers can only contain ascii characters.'
-            return dict(found=False)
+        if not py3compat.PY3:
+            try:
+                oname = oname.encode('ascii')
+                #print '2- oname: <%r>' % oname  # dbg
+            except UnicodeError:
+                print 'Python identifiers can only contain ascii characters.'
+                return dict(found=False)
 
         alias_ns = None
         if namespaces is None:
@@ -1701,7 +1703,8 b' class InteractiveShell(SingletonConfigurable, Magic):'
                                                         include_latest=True):
             if cell.strip(): # Ignore blank lines
                 for line in cell.splitlines():
-                    self.readline.add_history(line.encode(stdin_encoding, 'replace'))
+                    self.readline.add_history(py3compat.unicode_to_str(line,
+                                                                stdin_encoding))
 
     def set_next_input(self, s):
         """ Sets the 'default' input string for the next command line.
@@ -1907,8 +1910,6 b' class InteractiveShell(SingletonConfigurable, Magic):'
     
         self.define_magic('foo',foo_impl)
         """
-        
-        import new
         im = types.MethodType(func,self)
         old = getattr(self, "magic_" + magicname, None)
         setattr(self, "magic_" + magicname, im)
@@ -2175,15 +2176,10 b' class InteractiveShell(SingletonConfigurable, Magic):'
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
-        
-        if isinstance(fname, unicode):
-            # execfile uses default encoding instead of filesystem encoding
-            # so unicode filenames will fail
-            fname = fname.encode(sys.getfilesystemencoding() or sys.getdefaultencoding())
 
         with prepended_to_syspath(dname):
             try:
-                execfile(fname,*where)
+                py3compat.execfile(fname,*where)
             except SystemExit, status:
                 # If the call was made with 0 or None exit status (sys.exit(0)
                 # or sys.exit() ), don't bother showing a traceback, as both of

IPython/core/magic.py

@@ -15,7 +15,7 b''
 # Imports
 #-----------------------------------------------------------------------------
 
-import __builtin__
+import __builtin__ as builtin_mod
 import __future__
 import bdb
 import inspect
@@ -1729,7 +1729,7 b' Currently the magic system has the following functions:\\n"""'
             # Since this seems to be done by the interpreter itself, the best
             # we can do is to at least restore __builtins__ for the user on
             # exit.
-            self.shell.user_ns['__builtins__'] = __builtin__
+            self.shell.user_ns['__builtins__'] = builtin_mod
             
             # Ensure key global structures are restored
             sys.argv = save_argv

IPython/core/splitinput.py

@@ -22,6 +22,8 b' Authors:'
 import re
 import sys
 
+from IPython.utils import py3compat
+
 #-----------------------------------------------------------------------------
 # Main function
 #-----------------------------------------------------------------------------
@@ -56,11 +58,7 b' def split_user_input(line, pattern=None):'
     manner.
     """
     # We need to ensure that the rest of this routine deals only with unicode
-    if type(line)==str:
-        codec = sys.stdin.encoding
-        if codec is None:
-            codec = 'utf-8'
-        line = line.decode(codec)
+    line = py3compat.cast_unicode(line, sys.stdin.encoding or 'utf-8')
         
     if pattern is None:
         pattern = line_split
@@ -75,15 +73,16 b' def split_user_input(line, pattern=None):'
         pre = re.match('^(\s*)(.*)',line).groups()[0]
     else:
         pre,ifun,the_rest = match.groups()
-
-    # ifun has to be a valid python identifier, so it better encode into
-    # ascii.  We do still make it a unicode string so that we consistently
-    # return unicode, but it will be one that is guaranteed to be pure ascii
-    try:
-        ifun = unicode(ifun.encode('ascii'))
-    except UnicodeEncodeError:
-        the_rest = ifun + u' ' + the_rest
-        ifun = u''
+    
+    if not py3compat.PY3:
+        # ifun has to be a valid python identifier, so it better encode into
+        # ascii.  We do still make it a unicode string so that we consistently
+        # return unicode, but it will be one that is guaranteed to be pure ascii
+        try:
+            ifun = unicode(ifun.encode('ascii'))
+        except UnicodeEncodeError:
+            the_rest = ifun + u' ' + the_rest
+            ifun = u''
 
     #print 'line:<%s>' % line # dbg
     #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun.strip(),the_rest) # dbg

IPython/core/tests/tclass.py

@@ -2,6 +2,8 b''
 
 See test_run for details."""
 
+from __future__ import print_function
+
 import sys
 
 # We want to ensure that while objects remain available for immediate access,
@@ -10,10 +12,11 b' import sys'
 class C(object):
     def __init__(self,name):
         self.name = name
+        self.p = print
         self.flush_stdout = sys.stdout.flush
         
     def __del__(self):
-        print 'tclass.py: deleting object:',self.name
+        self.p('tclass.py: deleting object:',self.name)
         self.flush_stdout()
 
 try:

IPython/core/tests/test_application.py

@@ -6,6 +6,7 b' import tempfile'
 
 from IPython.core.application import BaseIPythonApplication
 from IPython.testing import decorators as testdec
+from IPython.utils import py3compat
 
 @testdec.onlyif_unicode_paths
 def test_unicode_cwd():
@@ -35,7 +36,7 b' def test_unicode_ipdir():'
     
     old_ipdir1 = os.environ.pop("IPYTHONDIR", None)
     old_ipdir2 = os.environ.pop("IPYTHON_DIR", None)
-    os.environ["IPYTHONDIR"] = ipdir.encode("utf-8")
+    os.environ["IPYTHONDIR"] = py3compat.unicode_to_str(ipdir, "utf-8")
     try:
         app = BaseIPythonApplication()
         # The lines below are copied from Application.initialize()

IPython/core/tests/test_compilerop.py

@@ -23,6 +23,7 b' import nose.tools as nt'
 
 # Our own imports
 from IPython.core import compilerop
+from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Test functions
@@ -51,7 +52,7 b' def test_cache():'
 def setUp():
     # Check we're in a proper Python 2 environment (some imports, such
     # as GTK, can change the default encoding, which can hide bugs.)
-    nt.assert_equal(sys.getdefaultencoding(), "ascii")
+    nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii")
 
 def test_cache_unicode():
     cp = compilerop.CachingCompiler()

IPython/core/tests/test_history.py

@@ -16,9 +16,10 b' import nose.tools as nt'
 # our own packages
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.core.history import HistoryManager, extract_hist_ranges
+from IPython.utils import py3compat
 
 def setUp():
-    nt.assert_equal(sys.getdefaultencoding(), "ascii")
+    nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii")
 
 def test_history():
     ip = get_ipython()

IPython/core/tests/test_inputsplitter.py

@@ -125,7 +125,7 b' def test_get_input_encoding():'
     nt.assert_true(isinstance(encoding, basestring))
     # simple-minded check that at least encoding a simple string works with the
     # encoding we got.
-    nt.assert_equal('test'.encode(encoding), 'test')
+    nt.assert_equal(u'test'.encode(encoding), b'test')
 
 
 class NoInputEncodingTestCase(unittest.TestCase):

IPython/core/tests/test_iplib.py

@@ -75,7 +75,7 b' Traceback (most recent call last):'
     div0()
   ...line 8, in div0
     x/y
-ZeroDivisionError: integer division or modulo by zero
+ZeroDivisionError: ...
     """
 
 
@@ -107,7 +107,7 b' ZeroDivisionError Traceback (most recent call last)'
       9 
      10 def sysexit(stat, mode):
 <BLANKLINE>
-ZeroDivisionError: integer division or modulo by zero
+ZeroDivisionError: ...
 """
 
 
@@ -144,7 +144,7 b' ZeroDivisionError Traceback (most recent call last)'
       9 
      10 def sysexit(stat, mode):
 <BLANKLINE>
-ZeroDivisionError: integer division or modulo by zero
+ZeroDivisionError: ...
       """
 
 

IPython/frontend/terminal/interactiveshell.py

@@ -332,7 +332,7 b' class TerminalInteractiveShell(InteractiveShell):'
             self.set_readline_completer()
         
         try:
-            line = self.raw_input_original(prompt).decode(self.stdin_encoding)
+            line = py3compat.str_to_unicode(self.raw_input_original(prompt))
         except ValueError:
             warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
                  " or sys.stdout.close()!\nExiting IPython!")

IPython/lib/demo.py

@@ -169,7 +169,6 b" print 'bye!'"
 #
 #*****************************************************************************
 
-import exceptions
 import os
 import re
 import shlex
@@ -182,7 +181,7 b' from IPython.utils.text import marquee'
 
 __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']
 
-class DemoError(exceptions.Exception): pass
+class DemoError(Exception): pass
 
 def re_mark(mark):
     return re.compile(r'^\s*#\s+<demo>\s+%s\s*$' % mark,re.MULTILINE)

IPython/testing/globalipapp.py

@@ -20,7 +20,7 b' from __future__ import print_function'
 #-----------------------------------------------------------------------------
 
 # stdlib
-import __builtin__
+import __builtin__ as builtin_mod
 import os
 import sys
 from types import MethodType
@@ -131,7 +131,7 b' class ipnsdict(dict):'
         # aggressive low-level cleaning of the execution namespace, we need to
         # correct for that ourselves, to ensure consitency with the 'real'
         # ipython.
-        self['__builtins__'] = __builtin__
+        self['__builtins__'] = builtin_mod
         
     def __delitem__(self, key):
         """Part of the test suite checks that we can release all

IPython/utils/_process_common.py

@@ -17,6 +17,8 b' of subprocess utilities, and it contains tools that are common to all of them.'
 import subprocess
 import sys
 
+from IPython.utils import py3compat
+
 #-----------------------------------------------------------------------------
 # Function definitions
 #-----------------------------------------------------------------------------
@@ -116,8 +118,8 b' def getoutput(cmd):'
 
     out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
     if out is None:
-        out = ''
-    return out
+        return ''
+    return py3compat.bytes_to_str(out)
 
 
 def getoutputerror(cmd):
@@ -138,5 +140,6 b' def getoutputerror(cmd):'
 
     out_err = process_handler(cmd, lambda p: p.communicate())
     if out_err is None:
-        out_err = '', ''
-    return out_err
+        return '', ''
+    out, err = out_err
+    return py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)

IPython/utils/path.py

@@ -23,6 +23,7 b' import IPython'
 from IPython.utils import warn
 from IPython.utils.process import system
 from IPython.utils.importstring import import_item
+from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Code
@@ -30,14 +31,6 b' from IPython.utils.importstring import import_item'
 
 fs_encoding = sys.getfilesystemencoding()
 
-def _cast_unicode(s, enc=None):
-    """Turn 8-bit strings into unicode."""
-    if isinstance(s, bytes):
-        enc = enc or sys.getdefaultencoding()
-        return s.decode(enc)
-    return s
-
-
 def _get_long_path_name(path):
     """Dummy no-op."""
     return path
@@ -203,7 +196,7 b' def get_home_dir():'
         root=os.path.abspath(root).rstrip('\\')
         if _writable_dir(os.path.join(root, '_ipython')):
             os.environ["IPYKITROOT"] = root
-        return _cast_unicode(root, fs_encoding)
+        return py3compat.cast_unicode(root, fs_encoding)
 
     if os.name == 'posix':
         # Linux, Unix, AIX, OS X
@@ -218,11 +211,11 b' def get_home_dir():'
             homedir = Popen('echo $HOME', shell=True, 
                             stdout=PIPE).communicate()[0].strip()
             if homedir:
-                return _cast_unicode(homedir, fs_encoding)
+                return py3compat.cast_unicode(homedir, fs_encoding)
             else:
                 raise HomeDirError('Undefined $HOME, IPython cannot proceed.')
         else:
-            return _cast_unicode(homedir, fs_encoding)
+            return py3compat.cast_unicode(homedir, fs_encoding)
     elif os.name == 'nt':
         # Now for win9x, XP, Vista, 7?
         # For some strange reason all of these return 'nt' for os.name.
@@ -236,7 +229,7 b' def get_home_dir():'
             pass
         else:
             if _writable_dir(homedir):
-                return _cast_unicode(homedir, fs_encoding)
+                return py3compat.cast_unicode(homedir, fs_encoding)
 
         # Now look for a local home directory
         try:
@@ -245,7 +238,7 b' def get_home_dir():'
             pass
         else:
             if _writable_dir(homedir):
-                return _cast_unicode(homedir, fs_encoding)
+                return py3compat.cast_unicode(homedir, fs_encoding)
 
         # Now the users profile directory
         try:
@@ -254,7 +247,7 b' def get_home_dir():'
             pass
         else:
             if _writable_dir(homedir):
-                return _cast_unicode(homedir, fs_encoding)
+                return py3compat.cast_unicode(homedir, fs_encoding)
 
         # Use the registry to get the 'My Documents' folder.
         try:
@@ -269,7 +262,7 b' def get_home_dir():'
             pass
         else:
             if _writable_dir(homedir):
-                return _cast_unicode(homedir, fs_encoding)
+                return py3compat.cast_unicode(homedir, fs_encoding)
 
         # A user with a lot of unix tools in win32 may have defined $HOME.
         # Try this as a last ditch option.
@@ -279,7 +272,7 b' def get_home_dir():'
             pass
         else:
             if _writable_dir(homedir):
-                return _cast_unicode(homedir, fs_encoding)
+                return py3compat.cast_unicode(homedir, fs_encoding)
 
         # If all else fails, raise HomeDirError
         raise HomeDirError('No valid home directory could be found')
@@ -302,7 +295,7 b' def get_xdg_dir():'
         # use ~/.config if not set OR empty
         xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
         if xdg and _writable_dir(xdg):
-            return _cast_unicode(xdg, fs_encoding)
+            return py3compat.cast_unicode(xdg, fs_encoding)
     
     return None
     
@@ -311,7 +304,7 b' def get_ipython_dir():'
     """Get the IPython directory for this platform and user.
     
     This uses the logic in `get_home_dir` to find the home directory
-    and the adds .ipython to the end of the path.
+    and then adds .ipython to the end of the path.
     """
     
     env = os.environ
@@ -356,13 +349,13 b' def get_ipython_dir():'
                         " using a temp directory."%parent)
             ipdir = tempfile.mkdtemp()
 
-    return _cast_unicode(ipdir, fs_encoding)
+    return py3compat.cast_unicode(ipdir, fs_encoding)
 
 
 def get_ipython_package_dir():
     """Get the base directory where IPython itself is installed."""
     ipdir = os.path.dirname(IPython.__file__)
-    return _cast_unicode(ipdir, fs_encoding)
+    return py3compat.cast_unicode(ipdir, fs_encoding)
 
 
 def get_ipython_module_path(module_str):
@@ -377,7 +370,7 b' def get_ipython_module_path(module_str):'
     mod = import_item(module_str)
     the_path = mod.__file__.replace('.pyc', '.py')
     the_path = the_path.replace('.pyo', '.py')
-    return _cast_unicode(the_path, fs_encoding)
+    return py3compat.cast_unicode(the_path, fs_encoding)
 
 
 def expand_path(s):
@@ -442,7 +435,7 b' def filehash(path):'
     """Make an MD5 hash of a file, ignoring any differences in line
     ending characters."""
     with open(path, "rU") as f:
-        return md5(f.read()).hexdigest()
+        return md5(py3compat.str_to_bytes(f.read())).hexdigest()
 
 # If the config is unmodified from the default, we'll just delete it.
 # These are consistent for 0.10.x, thankfully. We're not going to worry about

IPython/utils/py3compat.py

@@ -10,6 +10,16 b' def decode(s, encoding=None):'
 def encode(u, encoding=None):
     encoding = encoding or sys.stdin.encoding or sys.getdefaultencoding()
     return u.encode(encoding, "replace")
+    
+def cast_unicode(s, encoding=None):
+    if isinstance(s, bytes):
+        return decode(s, encoding)
+    return s
+
+def cast_bytes(s, encoding=None):
+    if not isinstance(s, bytes):
+        return encode(s, encoding)
+    return s
 
 if sys.version_info[0] >= 3:
     PY3 = True
@@ -33,5 +43,6 b' else:'
     str_to_bytes = no_code
     bytes_to_str = no_code
 
-def execfile(fname, glob, loc):
+def execfile(fname, glob, loc=None):
+    loc = loc if (loc is not None) else glob
     exec compile(open(fname).read(), fname, 'exec') in glob, loc