diff --git a/IPython/config/loader.py b/IPython/config/loader.py index 1e1620e..3cbd132 100644 --- a/IPython/config/loader.py +++ b/IPython/config/loader.py @@ -18,13 +18,13 @@ Authors # Imports #----------------------------------------------------------------------------- -import __builtin__ +import __builtin__ as builtin_mod import re import sys from IPython.external import argparse from IPython.utils.path import filefind, get_ipython_dir -from IPython.utils import warn +from IPython.utils import py3compat, warn #----------------------------------------------------------------------------- # Exceptions @@ -131,7 +131,7 @@ class Config(dict): # that you can't have section or attribute names that are # builtins. try: - return getattr(__builtin__, key) + return getattr(builtin_mod, key) except AttributeError: pass if is_section_key(key): @@ -146,7 +146,7 @@ class Config(dict): def __setitem__(self, key, value): # Don't allow names in __builtin__ to be modified. - if hasattr(__builtin__, key): + if hasattr(builtin_mod, key): raise ConfigError('Config variable names cannot have the same name ' 'as a Python builtin: %s' % key) if self._is_section_key(key): @@ -312,7 +312,7 @@ class PyFileConfigLoader(FileConfigLoader): namespace = dict(load_subconfig=load_subconfig, get_config=get_config) fs_encoding = sys.getfilesystemencoding() or 'ascii' conf_filename = self.full_filename.encode(fs_encoding) - execfile(conf_filename, namespace) + py3compat.execfile(conf_filename, namespace) def _convert_to_config(self): if self.data is None: @@ -586,13 +586,7 @@ class ArgParseConfigLoader(CommandLineConfigLoader): def _parse_args(self, args): """self.parser->self.parsed_data""" # decode sys.argv to support unicode command-line options - uargs = [] - for a in args: - if isinstance(a, str): - # don't decode if we already got unicode - a = a.decode(sys.stdin.encoding or - sys.getdefaultencoding()) - uargs.append(a) + uargs = [py3compat.cast_unicode(a) for a in args] self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs) def _convert_to_config(self): diff --git a/IPython/core/completerlib.py b/IPython/core/completerlib.py index 98ba2ec..15e5b7e 100644 --- a/IPython/core/completerlib.py +++ b/IPython/core/completerlib.py @@ -30,6 +30,7 @@ from zipimport import zipimporter # Our own imports from IPython.core.completer import expand_user, compress_user from IPython.core.error import TryNext +from IPython.utils import py3compat # FIXME: this should be pulled in with the right call via the component system from IPython.core.ipapi import get as get_ipython @@ -66,10 +67,9 @@ def shlex_split(x): # Example: # %run "c:/python -> ['%run','"c:/python'] - # shlex.split has unicode bugs, so encode first to str - if isinstance(x, unicode): - # don't raise errors on encoding: - x = x.encode(sys.stdin.encoding or sys.getdefaultencoding(), 'replace') + # shlex.split has unicode bugs in Python 2, so encode first to str + if not py3compat.PY3: + x = py3compat.cast_bytes(x) endofline = [] while x != '': diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index f61b348..58e77f0 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -17,7 +17,7 @@ from __future__ import with_statement from __future__ import absolute_import -import __builtin__ +import __builtin__ as builtin_mod import __future__ import abc import ast @@ -61,6 +61,7 @@ from IPython.core.profiledir import ProfileDir from IPython.external.Itpl import ItplNS from IPython.utils import PyColorize from IPython.utils import io +from IPython.utils import py3compat from IPython.utils.doctestreload import doctest_reload from IPython.utils.io import ask_yes_no, rprint from IPython.utils.ipstruct import Struct @@ -979,13 +980,13 @@ class InteractiveShell(SingletonConfigurable, Magic): # Set __name__ to __main__ to better match the behavior of the # normal interpreter. user_ns = {'__name__' :'__main__', - '__builtin__' : __builtin__, - '__builtins__' : __builtin__, + py3compat.builtin_mod_name: builtin_mod, + '__builtins__' : builtin_mod, } else: user_ns.setdefault('__name__','__main__') - user_ns.setdefault('__builtin__',__builtin__) - user_ns.setdefault('__builtins__',__builtin__) + user_ns.setdefault(py3compat.builtin_mod_name,builtin_mod) + user_ns.setdefault('__builtins__',builtin_mod) if user_global_ns is None: user_global_ns = user_ns @@ -1255,14 +1256,15 @@ class InteractiveShell(SingletonConfigurable, Magic): Has special code to detect magic functions. """ - #oname = oname.strip() + oname = oname.strip() #print '1- oname: <%r>' % oname # dbg - try: - oname = oname.strip().encode('ascii') - #print '2- oname: <%r>' % oname # dbg - except UnicodeEncodeError: - print 'Python identifiers can only contain ascii characters.' - return dict(found=False) + if not py3compat.PY3: + try: + oname = oname.encode('ascii') + #print '2- oname: <%r>' % oname # dbg + except UnicodeError: + print 'Python identifiers can only contain ascii characters.' + return dict(found=False) alias_ns = None if namespaces is None: @@ -1701,7 +1703,8 @@ class InteractiveShell(SingletonConfigurable, Magic): include_latest=True): if cell.strip(): # Ignore blank lines for line in cell.splitlines(): - self.readline.add_history(line.encode(stdin_encoding, 'replace')) + self.readline.add_history(py3compat.unicode_to_str(line, + stdin_encoding)) def set_next_input(self, s): """ Sets the 'default' input string for the next command line. @@ -1907,8 +1910,6 @@ class InteractiveShell(SingletonConfigurable, Magic): self.define_magic('foo',foo_impl) """ - - import new im = types.MethodType(func,self) old = getattr(self, "magic_" + magicname, None) setattr(self, "magic_" + magicname, im) @@ -2175,15 +2176,10 @@ class InteractiveShell(SingletonConfigurable, Magic): # behavior of running a script from the system command line, where # Python inserts the script's directory into sys.path dname = os.path.dirname(fname) - - if isinstance(fname, unicode): - # execfile uses default encoding instead of filesystem encoding - # so unicode filenames will fail - fname = fname.encode(sys.getfilesystemencoding() or sys.getdefaultencoding()) with prepended_to_syspath(dname): try: - execfile(fname,*where) + py3compat.execfile(fname,*where) except SystemExit, status: # If the call was made with 0 or None exit status (sys.exit(0) # or sys.exit() ), don't bother showing a traceback, as both of diff --git a/IPython/core/magic.py b/IPython/core/magic.py index 64cd121..650905f 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -15,7 +15,7 @@ # Imports #----------------------------------------------------------------------------- -import __builtin__ +import __builtin__ as builtin_mod import __future__ import bdb import inspect @@ -1729,7 +1729,7 @@ Currently the magic system has the following functions:\n""" # Since this seems to be done by the interpreter itself, the best # we can do is to at least restore __builtins__ for the user on # exit. - self.shell.user_ns['__builtins__'] = __builtin__ + self.shell.user_ns['__builtins__'] = builtin_mod # Ensure key global structures are restored sys.argv = save_argv diff --git a/IPython/core/splitinput.py b/IPython/core/splitinput.py index 5cdbb18..8546d91 100755 --- a/IPython/core/splitinput.py +++ b/IPython/core/splitinput.py @@ -22,6 +22,8 @@ Authors: import re import sys +from IPython.utils import py3compat + #----------------------------------------------------------------------------- # Main function #----------------------------------------------------------------------------- @@ -56,11 +58,7 @@ def split_user_input(line, pattern=None): manner. """ # We need to ensure that the rest of this routine deals only with unicode - if type(line)==str: - codec = sys.stdin.encoding - if codec is None: - codec = 'utf-8' - line = line.decode(codec) + line = py3compat.cast_unicode(line, sys.stdin.encoding or 'utf-8') if pattern is None: pattern = line_split @@ -75,15 +73,16 @@ def split_user_input(line, pattern=None): pre = re.match('^(\s*)(.*)',line).groups()[0] else: pre,ifun,the_rest = match.groups() - - # ifun has to be a valid python identifier, so it better encode into - # ascii. We do still make it a unicode string so that we consistently - # return unicode, but it will be one that is guaranteed to be pure ascii - try: - ifun = unicode(ifun.encode('ascii')) - except UnicodeEncodeError: - the_rest = ifun + u' ' + the_rest - ifun = u'' + + if not py3compat.PY3: + # ifun has to be a valid python identifier, so it better encode into + # ascii. We do still make it a unicode string so that we consistently + # return unicode, but it will be one that is guaranteed to be pure ascii + try: + ifun = unicode(ifun.encode('ascii')) + except UnicodeEncodeError: + the_rest = ifun + u' ' + the_rest + ifun = u'' #print 'line:<%s>' % line # dbg #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun.strip(),the_rest) # dbg diff --git a/IPython/core/tests/tclass.py b/IPython/core/tests/tclass.py index 38b3a5a..8d4009a 100644 --- a/IPython/core/tests/tclass.py +++ b/IPython/core/tests/tclass.py @@ -2,6 +2,8 @@ See test_run for details.""" +from __future__ import print_function + import sys # We want to ensure that while objects remain available for immediate access, @@ -10,10 +12,11 @@ import sys class C(object): def __init__(self,name): self.name = name + self.p = print self.flush_stdout = sys.stdout.flush def __del__(self): - print 'tclass.py: deleting object:',self.name + self.p('tclass.py: deleting object:',self.name) self.flush_stdout() try: diff --git a/IPython/core/tests/test_application.py b/IPython/core/tests/test_application.py index 3a59a0f..a1d0474 100644 --- a/IPython/core/tests/test_application.py +++ b/IPython/core/tests/test_application.py @@ -6,6 +6,7 @@ import tempfile from IPython.core.application import BaseIPythonApplication from IPython.testing import decorators as testdec +from IPython.utils import py3compat @testdec.onlyif_unicode_paths def test_unicode_cwd(): @@ -35,7 +36,7 @@ def test_unicode_ipdir(): old_ipdir1 = os.environ.pop("IPYTHONDIR", None) old_ipdir2 = os.environ.pop("IPYTHON_DIR", None) - os.environ["IPYTHONDIR"] = ipdir.encode("utf-8") + os.environ["IPYTHONDIR"] = py3compat.unicode_to_str(ipdir, "utf-8") try: app = BaseIPythonApplication() # The lines below are copied from Application.initialize() diff --git a/IPython/core/tests/test_compilerop.py b/IPython/core/tests/test_compilerop.py index e9989db..0854b54 100644 --- a/IPython/core/tests/test_compilerop.py +++ b/IPython/core/tests/test_compilerop.py @@ -23,6 +23,7 @@ import nose.tools as nt # Our own imports from IPython.core import compilerop +from IPython.utils import py3compat #----------------------------------------------------------------------------- # Test functions @@ -51,7 +52,7 @@ def test_cache(): def setUp(): # Check we're in a proper Python 2 environment (some imports, such # as GTK, can change the default encoding, which can hide bugs.) - nt.assert_equal(sys.getdefaultencoding(), "ascii") + nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii") def test_cache_unicode(): cp = compilerop.CachingCompiler() diff --git a/IPython/core/tests/test_history.py b/IPython/core/tests/test_history.py index 1cbacd2..f9536d7 100644 --- a/IPython/core/tests/test_history.py +++ b/IPython/core/tests/test_history.py @@ -16,9 +16,10 @@ import nose.tools as nt # our own packages from IPython.utils.tempdir import TemporaryDirectory from IPython.core.history import HistoryManager, extract_hist_ranges +from IPython.utils import py3compat def setUp(): - nt.assert_equal(sys.getdefaultencoding(), "ascii") + nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii") def test_history(): ip = get_ipython() diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py index 77cbd2a..639a5eb 100644 --- a/IPython/core/tests/test_inputsplitter.py +++ b/IPython/core/tests/test_inputsplitter.py @@ -125,7 +125,7 @@ def test_get_input_encoding(): nt.assert_true(isinstance(encoding, basestring)) # simple-minded check that at least encoding a simple string works with the # encoding we got. - nt.assert_equal('test'.encode(encoding), 'test') + nt.assert_equal(u'test'.encode(encoding), b'test') class NoInputEncodingTestCase(unittest.TestCase): diff --git a/IPython/core/tests/test_iplib.py b/IPython/core/tests/test_iplib.py index cb91ab2..dba13a5 100644 --- a/IPython/core/tests/test_iplib.py +++ b/IPython/core/tests/test_iplib.py @@ -75,7 +75,7 @@ Traceback (most recent call last): div0() ...line 8, in div0 x/y -ZeroDivisionError: integer division or modulo by zero +ZeroDivisionError: ... """ @@ -107,7 +107,7 @@ ZeroDivisionError Traceback (most recent call last) 9 10 def sysexit(stat, mode): -ZeroDivisionError: integer division or modulo by zero +ZeroDivisionError: ... """ @@ -144,7 +144,7 @@ ZeroDivisionError Traceback (most recent call last) 9 10 def sysexit(stat, mode): -ZeroDivisionError: integer division or modulo by zero +ZeroDivisionError: ... """ diff --git a/IPython/frontend/terminal/interactiveshell.py b/IPython/frontend/terminal/interactiveshell.py index 253f3e4..f265c73 100644 --- a/IPython/frontend/terminal/interactiveshell.py +++ b/IPython/frontend/terminal/interactiveshell.py @@ -332,7 +332,7 @@ class TerminalInteractiveShell(InteractiveShell): self.set_readline_completer() try: - line = self.raw_input_original(prompt).decode(self.stdin_encoding) + line = py3compat.str_to_unicode(self.raw_input_original(prompt)) except ValueError: warn("\n********\nYou or a %run:ed script called sys.stdin.close()" " or sys.stdout.close()!\nExiting IPython!") diff --git a/IPython/lib/demo.py b/IPython/lib/demo.py index e2bc7c5..e11e339 100644 --- a/IPython/lib/demo.py +++ b/IPython/lib/demo.py @@ -169,7 +169,6 @@ print 'bye!' # #***************************************************************************** -import exceptions import os import re import shlex @@ -182,7 +181,7 @@ from IPython.utils.text import marquee __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError'] -class DemoError(exceptions.Exception): pass +class DemoError(Exception): pass def re_mark(mark): return re.compile(r'^\s*#\s+\s+%s\s*$' % mark,re.MULTILINE) diff --git a/IPython/testing/globalipapp.py b/IPython/testing/globalipapp.py index 89b5f27..655bd64 100644 --- a/IPython/testing/globalipapp.py +++ b/IPython/testing/globalipapp.py @@ -20,7 +20,7 @@ from __future__ import print_function #----------------------------------------------------------------------------- # stdlib -import __builtin__ +import __builtin__ as builtin_mod import os import sys from types import MethodType @@ -131,7 +131,7 @@ class ipnsdict(dict): # aggressive low-level cleaning of the execution namespace, we need to # correct for that ourselves, to ensure consitency with the 'real' # ipython. - self['__builtins__'] = __builtin__ + self['__builtins__'] = builtin_mod def __delitem__(self, key): """Part of the test suite checks that we can release all diff --git a/IPython/utils/_process_common.py b/IPython/utils/_process_common.py index a4a0fd8..66eb0d0 100644 --- a/IPython/utils/_process_common.py +++ b/IPython/utils/_process_common.py @@ -17,6 +17,8 @@ of subprocess utilities, and it contains tools that are common to all of them. import subprocess import sys +from IPython.utils import py3compat + #----------------------------------------------------------------------------- # Function definitions #----------------------------------------------------------------------------- @@ -116,8 +118,8 @@ def getoutput(cmd): out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT) if out is None: - out = '' - return out + return '' + return py3compat.bytes_to_str(out) def getoutputerror(cmd): @@ -138,5 +140,6 @@ def getoutputerror(cmd): out_err = process_handler(cmd, lambda p: p.communicate()) if out_err is None: - out_err = '', '' - return out_err + return '', '' + out, err = out_err + return py3compat.bytes_to_str(out), py3compat.bytes_to_str(err) diff --git a/IPython/utils/path.py b/IPython/utils/path.py index ee8d0e7..0e5b9df 100644 --- a/IPython/utils/path.py +++ b/IPython/utils/path.py @@ -23,6 +23,7 @@ import IPython from IPython.utils import warn from IPython.utils.process import system from IPython.utils.importstring import import_item +from IPython.utils import py3compat #----------------------------------------------------------------------------- # Code @@ -30,14 +31,6 @@ from IPython.utils.importstring import import_item fs_encoding = sys.getfilesystemencoding() -def _cast_unicode(s, enc=None): - """Turn 8-bit strings into unicode.""" - if isinstance(s, bytes): - enc = enc or sys.getdefaultencoding() - return s.decode(enc) - return s - - def _get_long_path_name(path): """Dummy no-op.""" return path @@ -203,7 +196,7 @@ def get_home_dir(): root=os.path.abspath(root).rstrip('\\') if _writable_dir(os.path.join(root, '_ipython')): os.environ["IPYKITROOT"] = root - return _cast_unicode(root, fs_encoding) + return py3compat.cast_unicode(root, fs_encoding) if os.name == 'posix': # Linux, Unix, AIX, OS X @@ -218,11 +211,11 @@ def get_home_dir(): homedir = Popen('echo $HOME', shell=True, stdout=PIPE).communicate()[0].strip() if homedir: - return _cast_unicode(homedir, fs_encoding) + return py3compat.cast_unicode(homedir, fs_encoding) else: raise HomeDirError('Undefined $HOME, IPython cannot proceed.') else: - return _cast_unicode(homedir, fs_encoding) + return py3compat.cast_unicode(homedir, fs_encoding) elif os.name == 'nt': # Now for win9x, XP, Vista, 7? # For some strange reason all of these return 'nt' for os.name. @@ -236,7 +229,7 @@ def get_home_dir(): pass else: if _writable_dir(homedir): - return _cast_unicode(homedir, fs_encoding) + return py3compat.cast_unicode(homedir, fs_encoding) # Now look for a local home directory try: @@ -245,7 +238,7 @@ def get_home_dir(): pass else: if _writable_dir(homedir): - return _cast_unicode(homedir, fs_encoding) + return py3compat.cast_unicode(homedir, fs_encoding) # Now the users profile directory try: @@ -254,7 +247,7 @@ def get_home_dir(): pass else: if _writable_dir(homedir): - return _cast_unicode(homedir, fs_encoding) + return py3compat.cast_unicode(homedir, fs_encoding) # Use the registry to get the 'My Documents' folder. try: @@ -269,7 +262,7 @@ def get_home_dir(): pass else: if _writable_dir(homedir): - return _cast_unicode(homedir, fs_encoding) + return py3compat.cast_unicode(homedir, fs_encoding) # A user with a lot of unix tools in win32 may have defined $HOME. # Try this as a last ditch option. @@ -279,7 +272,7 @@ def get_home_dir(): pass else: if _writable_dir(homedir): - return _cast_unicode(homedir, fs_encoding) + return py3compat.cast_unicode(homedir, fs_encoding) # If all else fails, raise HomeDirError raise HomeDirError('No valid home directory could be found') @@ -302,7 +295,7 @@ def get_xdg_dir(): # use ~/.config if not set OR empty xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config') if xdg and _writable_dir(xdg): - return _cast_unicode(xdg, fs_encoding) + return py3compat.cast_unicode(xdg, fs_encoding) return None @@ -311,7 +304,7 @@ def get_ipython_dir(): """Get the IPython directory for this platform and user. This uses the logic in `get_home_dir` to find the home directory - and the adds .ipython to the end of the path. + and then adds .ipython to the end of the path. """ env = os.environ @@ -356,13 +349,13 @@ def get_ipython_dir(): " using a temp directory."%parent) ipdir = tempfile.mkdtemp() - return _cast_unicode(ipdir, fs_encoding) + return py3compat.cast_unicode(ipdir, fs_encoding) def get_ipython_package_dir(): """Get the base directory where IPython itself is installed.""" ipdir = os.path.dirname(IPython.__file__) - return _cast_unicode(ipdir, fs_encoding) + return py3compat.cast_unicode(ipdir, fs_encoding) def get_ipython_module_path(module_str): @@ -377,7 +370,7 @@ def get_ipython_module_path(module_str): mod = import_item(module_str) the_path = mod.__file__.replace('.pyc', '.py') the_path = the_path.replace('.pyo', '.py') - return _cast_unicode(the_path, fs_encoding) + return py3compat.cast_unicode(the_path, fs_encoding) def expand_path(s): @@ -442,7 +435,7 @@ def filehash(path): """Make an MD5 hash of a file, ignoring any differences in line ending characters.""" with open(path, "rU") as f: - return md5(f.read()).hexdigest() + return md5(py3compat.str_to_bytes(f.read())).hexdigest() # If the config is unmodified from the default, we'll just delete it. # These are consistent for 0.10.x, thankfully. We're not going to worry about diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py index 10fd30e..e491de7 100644 --- a/IPython/utils/py3compat.py +++ b/IPython/utils/py3compat.py @@ -10,6 +10,16 @@ def decode(s, encoding=None): def encode(u, encoding=None): encoding = encoding or sys.stdin.encoding or sys.getdefaultencoding() return u.encode(encoding, "replace") + +def cast_unicode(s, encoding=None): + if isinstance(s, bytes): + return decode(s, encoding) + return s + +def cast_bytes(s, encoding=None): + if not isinstance(s, bytes): + return encode(s, encoding) + return s if sys.version_info[0] >= 3: PY3 = True @@ -33,5 +43,6 @@ else: str_to_bytes = no_code bytes_to_str = no_code -def execfile(fname, glob, loc): +def execfile(fname, glob, loc=None): + loc = loc if (loc is not None) else glob exec compile(open(fname).read(), fname, 'exec') in glob, loc