Merge pull request #6184 from takluyver/rm-py3compat-open

Remove py3compat.open

docs/source/whatsnew/pr/incompat-py3compat-open.rst

@@ -0,0 +1,2 b''
+* ``IPython.utils.py3compat.open`` was removed: :func:`io.open` provides all
+  the same functionality.

IPython/core/tests/test_history.py

@@ -1,187 +1,188 b''
 # coding: utf-8
 """Tests for the IPython tab-completion machinery.
 """
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 # stdlib
+import io
 import os
 import sys
 import tempfile
 from datetime import datetime
 
 # third party
 import nose.tools as nt
 
 # our own packages
 from IPython.config.loader import Config
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.core.history import HistoryManager, extract_hist_ranges
 from IPython.utils import py3compat
 
 def setUp():
     nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii")
 
 def test_history():
     ip = get_ipython()
     with TemporaryDirectory() as tmpdir:
         hist_manager_ori = ip.history_manager
         hist_file = os.path.join(tmpdir, 'history.sqlite')
         try:
             ip.history_manager = HistoryManager(shell=ip, hist_file=hist_file)
             hist = [u'a=1', u'def f():\n    test = 1\n    return test', u"b='€Æ¾÷ß'"]
             for i, h in enumerate(hist, start=1):
                 ip.history_manager.store_inputs(i, h)
 
             ip.history_manager.db_log_output = True
             # Doesn't match the input, but we'll just check it's stored.
             ip.history_manager.output_hist_reprs[3] = "spam"
             ip.history_manager.store_output(3)
 
             nt.assert_equal(ip.history_manager.input_hist_raw, [''] + hist)
             
             # Detailed tests for _get_range_session
             grs = ip.history_manager._get_range_session
             nt.assert_equal(list(grs(start=2,stop=-1)), list(zip([0], [2], hist[1:-1])))
             nt.assert_equal(list(grs(start=-2)), list(zip([0,0], [2,3], hist[-2:])))
             nt.assert_equal(list(grs(output=True)), list(zip([0,0,0], [1,2,3], zip(hist, [None,None,'spam']))))
 
             # Check whether specifying a range beyond the end of the current
             # session results in an error (gh-804)
             ip.magic('%hist 2-500')
             
             # Check that we can write non-ascii characters to a file
             ip.magic("%%hist -f %s" % os.path.join(tmpdir, "test1"))
             ip.magic("%%hist -pf %s" % os.path.join(tmpdir, "test2"))
             ip.magic("%%hist -nf %s" % os.path.join(tmpdir, "test3"))
             ip.magic("%%save %s 1-10" % os.path.join(tmpdir, "test4"))
 
             # New session
             ip.history_manager.reset()
             newcmds = [u"z=5",
                        u"class X(object):\n    pass",
                        u"k='p'",
                        u"z=5"]
             for i, cmd in enumerate(newcmds, start=1):
                 ip.history_manager.store_inputs(i, cmd)
             gothist = ip.history_manager.get_range(start=1, stop=4)
             nt.assert_equal(list(gothist), list(zip([0,0,0],[1,2,3], newcmds)))
             # Previous session:
             gothist = ip.history_manager.get_range(-1, 1, 4)
             nt.assert_equal(list(gothist), list(zip([1,1,1],[1,2,3], hist)))
 
             newhist = [(2, i, c) for (i, c) in enumerate(newcmds, 1)]
 
             # Check get_hist_tail
             gothist = ip.history_manager.get_tail(5, output=True,
                                                     include_latest=True)
             expected = [(1, 3, (hist[-1], "spam"))] \
                 + [(s, n, (c, None)) for (s, n, c) in newhist]
             nt.assert_equal(list(gothist), expected)
 
             gothist = ip.history_manager.get_tail(2)
             expected = newhist[-3:-1]
             nt.assert_equal(list(gothist), expected)
 
             # Check get_hist_search
             gothist = ip.history_manager.search("*test*")
             nt.assert_equal(list(gothist), [(1,2,hist[1])] )
 
             gothist = ip.history_manager.search("*=*")
             nt.assert_equal(list(gothist),
                             [(1, 1, hist[0]),
                              (1, 2, hist[1]),
                              (1, 3, hist[2]),
                              newhist[0],
                              newhist[2],
                              newhist[3]])
 
             gothist = ip.history_manager.search("*=*", n=4)
             nt.assert_equal(list(gothist),
                             [(1, 3, hist[2]),
                              newhist[0],
                              newhist[2],
                              newhist[3]])
 
             gothist = ip.history_manager.search("*=*", unique=True)
             nt.assert_equal(list(gothist),
                             [(1, 1, hist[0]),
                              (1, 2, hist[1]),
                              (1, 3, hist[2]),
                              newhist[2],
                              newhist[3]])
 
             gothist = ip.history_manager.search("*=*", unique=True, n=3)
             nt.assert_equal(list(gothist),
                             [(1, 3, hist[2]),
                              newhist[2],
                              newhist[3]])
 
             gothist = ip.history_manager.search("b*", output=True)
             nt.assert_equal(list(gothist), [(1,3,(hist[2],"spam"))] )
 
             # Cross testing: check that magic %save can get previous session.
             testfilename = os.path.realpath(os.path.join(tmpdir, "test.py"))
             ip.magic("save " + testfilename + " ~1/1-3")
-            with py3compat.open(testfilename, encoding='utf-8') as testfile:
+            with io.open(testfilename, encoding='utf-8') as testfile:
                 nt.assert_equal(testfile.read(),
                                         u"# coding: utf-8\n" + u"\n".join(hist)+u"\n")
 
             # Duplicate line numbers - check that it doesn't crash, and
             # gets a new session
             ip.history_manager.store_inputs(1, "rogue")
             ip.history_manager.writeout_cache()
             nt.assert_equal(ip.history_manager.session_number, 3)
         finally:
             # Ensure saving thread is shut down before we try to clean up the files
             ip.history_manager.save_thread.stop()
             # Forcibly close database rather than relying on garbage collection
             ip.history_manager.db.close()
             # Restore history manager
             ip.history_manager = hist_manager_ori
 
 
 def test_extract_hist_ranges():
     instr = "1 2/3 ~4/5-6 ~4/7-~4/9 ~9/2-~7/5 ~10/"
     expected = [(0, 1, 2),  # 0 == current session
                 (2, 3, 4),
                 (-4, 5, 7),
                 (-4, 7, 10),
                 (-9, 2, None),  # None == to end
                 (-8, 1, None),
                 (-7, 1, 6),
                 (-10, 1, None)]
     actual = list(extract_hist_ranges(instr))
     nt.assert_equal(actual, expected)
 
 def test_magic_rerun():
     """Simple test for %rerun (no args -> rerun last line)"""
     ip = get_ipython()
     ip.run_cell("a = 10", store_history=True)
     ip.run_cell("a += 1", store_history=True)
     nt.assert_equal(ip.user_ns["a"], 11)
     ip.run_cell("%rerun", store_history=True)
     nt.assert_equal(ip.user_ns["a"], 12)
 
 def test_timestamp_type():
     ip = get_ipython()
     info = ip.history_manager.get_session_info()
     nt.assert_true(isinstance(info[1], datetime))
 
 def test_hist_file_config():
     cfg = Config()
     tfile = tempfile.NamedTemporaryFile(delete=False)
     cfg.HistoryManager.hist_file = tfile.name
     try:
         hm = HistoryManager(shell=get_ipython(), config=cfg)
         nt.assert_equal(hm.hist_file, cfg.HistoryManager.hist_file)
     finally:
         try:
             os.remove(tfile.name)
         except OSError:
             # same catch as in testing.tools.TempFileMixin
             # On Windows, even though we close the file, we still can't
             # delete it.  I have no clue why
             pass
 

IPython/utils/py3compat.py

@@ -1,271 +1,247 b''
 # coding: utf-8
 """Compatibility tricks for Python 3. Mainly to do with unicode."""
 import functools
 import os
 import sys
 import re
 import types
 
 from .encoding import DEFAULT_ENCODING
 
-orig_open = open
-
 def no_code(x, encoding=None):
     return x
 
 def decode(s, encoding=None):
     encoding = encoding or DEFAULT_ENCODING
     return s.decode(encoding, "replace")
 
 def encode(u, encoding=None):
     encoding = encoding or DEFAULT_ENCODING
     return u.encode(encoding, "replace")
 
 
 def cast_unicode(s, encoding=None):
     if isinstance(s, bytes):
         return decode(s, encoding)
     return s
 
 def cast_bytes(s, encoding=None):
     if not isinstance(s, bytes):
         return encode(s, encoding)
     return s
 
 def _modify_str_or_docstring(str_change_func):
     @functools.wraps(str_change_func)
     def wrapper(func_or_str):
         if isinstance(func_or_str, string_types):
             func = None
             doc = func_or_str
         else:
             func = func_or_str
             doc = func.__doc__
         
         doc = str_change_func(doc)
         
         if func:
             func.__doc__ = doc
             return func
         return doc
     return wrapper
 
 def safe_unicode(e):
     """unicode(e) with various fallbacks. Used for exceptions, which may not be
     safe to call unicode() on.
     """
     try:
         return unicode_type(e)
     except UnicodeError:
         pass
 
     try:
         return str_to_unicode(str(e))
     except UnicodeError:
         pass
 
     try:
         return str_to_unicode(repr(e))
     except UnicodeError:
         pass
 
     return u'Unrecoverably corrupt evalue'
 
 if sys.version_info[0] >= 3:
     PY3 = True
     
     # keep reference to builtin_mod because the kernel overrides that value
     # to forward requests to a frontend.
     def input(prompt=''):
         return builtin_mod.input(prompt)
     
     builtin_mod_name = "builtins"
     import builtins as builtin_mod
     
     str_to_unicode = no_code
     unicode_to_str = no_code
     str_to_bytes = encode
     bytes_to_str = decode
     cast_bytes_py2 = no_code
     cast_unicode_py2 = no_code
     
     string_types = (str,)
     unicode_type = str
     
     def isidentifier(s, dotted=False):
         if dotted:
             return all(isidentifier(a) for a in s.split("."))
         return s.isidentifier()
-    
-    open = orig_open
+
     xrange = range
     def iteritems(d): return iter(d.items())
     def itervalues(d): return iter(d.values())
     getcwd = os.getcwd
     
     MethodType = types.MethodType
     
     def execfile(fname, glob, loc=None):
         loc = loc if (loc is not None) else glob
         with open(fname, 'rb') as f:
             exec(compile(f.read(), fname, 'exec'), glob, loc)
     
     # Refactor print statements in doctests.
     _print_statement_re = re.compile(r"\bprint (?P<expr>.*)$", re.MULTILINE)
     def _print_statement_sub(match):
         expr = match.groups('expr')
         return "print(%s)" % expr
     
     @_modify_str_or_docstring
     def doctest_refactor_print(doc):
         """Refactor 'print x' statements in a doctest to print(x) style. 2to3
         unfortunately doesn't pick up on our doctests.
         
         Can accept a string or a function, so it can be used as a decorator."""
         return _print_statement_re.sub(_print_statement_sub, doc)
     
     # Abstract u'abc' syntax:
     @_modify_str_or_docstring
     def u_format(s):
         """"{u}'abc'" --> "'abc'" (Python 3)
         
         Accepts a string or a function, so it can be used as a decorator."""
         return s.format(u='')
     
     def get_closure(f):
         """Get a function's closure attribute"""
         return f.__closure__
 
 else:
     PY3 = False
     
     # keep reference to builtin_mod because the kernel overrides that value
     # to forward requests to a frontend.
     def input(prompt=''):
         return builtin_mod.raw_input(prompt)
     
     builtin_mod_name = "__builtin__"
     import __builtin__ as builtin_mod
     
     str_to_unicode = decode
     unicode_to_str = encode
     str_to_bytes = no_code
     bytes_to_str = no_code
     cast_bytes_py2 = cast_bytes
     cast_unicode_py2 = cast_unicode
     
     string_types = (str, unicode)
     unicode_type = unicode
     
     import re
     _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")
     def isidentifier(s, dotted=False):
         if dotted:
             return all(isidentifier(a) for a in s.split("."))
         return bool(_name_re.match(s))
     
-    class open(object):
-        """Wrapper providing key part of Python 3 open() interface."""
-        def __init__(self, fname, mode="r", encoding="utf-8"):
-            self.f = orig_open(fname, mode)
-            self.enc = encoding
-        
-        def write(self, s):
-            return self.f.write(s.encode(self.enc))
-        
-        def read(self, size=-1):
-            return self.f.read(size).decode(self.enc)
-        
-        def close(self):
-            return self.f.close()
-        
-        def __enter__(self):
-            return self
-        
-        def __exit__(self, etype, value, traceback):
-            self.f.close()
-    
     xrange = xrange
     def iteritems(d): return d.iteritems()
     def itervalues(d): return d.itervalues()
     getcwd = os.getcwdu
 
     def MethodType(func, instance):
         return types.MethodType(func, instance, type(instance))
     
     def doctest_refactor_print(func_or_str):
         return func_or_str
 
     def get_closure(f):
         """Get a function's closure attribute"""
         return f.func_closure
 
     # Abstract u'abc' syntax:
     @_modify_str_or_docstring
     def u_format(s):
         """"{u}'abc'" --> "u'abc'" (Python 2)
         
         Accepts a string or a function, so it can be used as a decorator."""
         return s.format(u='u')
 
     if sys.platform == 'win32':
         def execfile(fname, glob=None, loc=None):
             loc = loc if (loc is not None) else glob
             # The rstrip() is necessary b/c trailing whitespace in files will
             # cause an IndentationError in Python 2.6 (this was fixed in 2.7,
             # but we still support 2.6).  See issue 1027.
             scripttext = builtin_mod.open(fname).read().rstrip() + '\n'
             # compile converts unicode filename to str assuming
             # ascii. Let's do the conversion before calling compile
             if isinstance(fname, unicode):
                 filename = unicode_to_str(fname)
             else:
                 filename = fname
             exec(compile(scripttext, filename, 'exec'), glob, loc)
     else:
         def execfile(fname, *where):
             if isinstance(fname, unicode):
                 filename = fname.encode(sys.getfilesystemencoding())
             else:
                 filename = fname
             builtin_mod.execfile(filename, *where)
 
 
 def annotate(**kwargs):
     """Python 3 compatible function annotation for Python 2."""
     if not kwargs:
         raise ValueError('annotations must be provided as keyword arguments')
     def dec(f):
         if hasattr(f, '__annotations__'):
             for k, v in kwargs.items():
                 f.__annotations__[k] = v
         else:
             f.__annotations__ = kwargs
         return f
     return dec
 
 
 # Parts below taken from six:
 # Copyright (c) 2010-2013 Benjamin Peterson
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in all
 # copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
 def with_metaclass(meta, *bases):
     """Create a base class with a metaclass."""
     return meta("_NewBase", bases, {})

docs/autogen_api.py

@@ -1,64 +1,66 b''
 #!/usr/bin/env python
 """Script to auto-generate our API docs.
 """
 # stdlib imports
 import os
 import sys
 
 # local imports
 sys.path.append(os.path.abspath('sphinxext'))
 from apigen import ApiDocWriter
 
 #*****************************************************************************
 if __name__ == '__main__':
     pjoin = os.path.join
     package = 'IPython'
     outdir = pjoin('source','api','generated')
     docwriter = ApiDocWriter(package,rst_extension='.rst')
     # You have to escape the . here because . is a special char for regexps.
     # You must do make clean if you change this!
     docwriter.package_skip_patterns += [r'\.external$',
                                         # Extensions are documented elsewhere.
                                         r'\.extensions',
                                         r'\.config\.profile',
                                         # These should be accessed via nbformat.current
                                         r'\.nbformat\.v\d+',
                                         ]
 
     # The inputhook* modules often cause problems on import, such as trying to
     # load incompatible Qt bindings. It's easiest to leave them all out. The
     # main API is in the inputhook module, which is documented.
     docwriter.module_skip_patterns += [ r'\.lib\.inputhook.+',
                                         r'\.ipdoctest',
                                         r'\.testing\.plugin',
                                         # This just prints a deprecation msg:
                                         r'\.frontend$',
                                         # Deprecated:
-                                        r'\.core\.magics\.deprecated'
+                                        r'\.core\.magics\.deprecated',
                                         # We document this manually.
                                         r'\.utils\.py3compat',
                                         # These are exposed by nbformat.current
                                         r'\.nbformat\.convert',
                                         r'\.nbformat\.validator',
                                         # These are exposed in display
                                         r'\.core\.display',
                                         r'\.lib\.display',
+                                        # This isn't actually a module
+                                        r'\.html\.fabfile',
                                         ]
 
     # These modules import functions and classes from other places to expose
     # them as part of the public API. They must have __all__ defined. The
     # non-API modules they import from should be excluded by the skip patterns
     # above.
     docwriter.names_from__all__.update({
         'IPython.nbformat.current',
         'IPython.display',
     })
     
     # Now, generate the outputs
     docwriter.write_api_docs(outdir)
     # Write index with .txt extension - we can include it, but Sphinx won't try
     # to compile it
     docwriter.write_index(outdir, 'gen.txt',
                           relative_to = pjoin('source','api')
                           )
     print ('%d files written' % len(docwriter.written_modules))