Merge pull request #8240 from minrk/split-io

split utils.io

IPython/html/services/contents/tests/test_fileio.py

@@ -0,0 +1,131 b''
+# encoding: utf-8
+"""Tests for file IO"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import io as stdlib_io
+import os.path
+import stat
+
+import nose.tools as nt
+
+from IPython.testing.decorators import skip_win32
+from ..fileio import atomic_writing
+
+from IPython.utils.tempdir import TemporaryDirectory
+
+umask = 0
+
+def test_atomic_writing():
+    class CustomExc(Exception): pass
+
+    with TemporaryDirectory() as td:
+        f1 = os.path.join(td, 'penguin')
+        with stdlib_io.open(f1, 'w') as f:
+            f.write(u'Before')
+        
+        if os.name != 'nt':
+            os.chmod(f1, 0o701)
+            orig_mode = stat.S_IMODE(os.stat(f1).st_mode)
+
+        f2 = os.path.join(td, 'flamingo')
+        try:
+            os.symlink(f1, f2)
+            have_symlink = True
+        except (AttributeError, NotImplementedError, OSError):
+            # AttributeError: Python doesn't support it
+            # NotImplementedError: The system doesn't support it
+            # OSError: The user lacks the privilege (Windows)
+            have_symlink = False
+
+        with nt.assert_raises(CustomExc):
+            with atomic_writing(f1) as f:
+                f.write(u'Failing write')
+                raise CustomExc
+
+        # Because of the exception, the file should not have been modified
+        with stdlib_io.open(f1, 'r') as f:
+            nt.assert_equal(f.read(), u'Before')
+
+        with atomic_writing(f1) as f:
+            f.write(u'Overwritten')
+
+        with stdlib_io.open(f1, 'r') as f:
+            nt.assert_equal(f.read(), u'Overwritten')
+
+        if os.name != 'nt':
+            mode = stat.S_IMODE(os.stat(f1).st_mode)
+            nt.assert_equal(mode, orig_mode)
+
+        if have_symlink:
+            # Check that writing over a file preserves a symlink
+            with atomic_writing(f2) as f:
+                f.write(u'written from symlink')
+            
+            with stdlib_io.open(f1, 'r') as f:
+                nt.assert_equal(f.read(), u'written from symlink')
+
+def _save_umask():
+    global umask
+    umask = os.umask(0)
+    os.umask(umask)
+
+def _restore_umask():
+    os.umask(umask)
+
+@skip_win32
[e-mail]_setup(_save_umask, _restore_umask)
+def test_atomic_writing_umask():
+    with TemporaryDirectory() as td:
+        os.umask(0o022)
+        f1 = os.path.join(td, '1')
+        with atomic_writing(f1) as f:
+            f.write(u'1')
+        mode = stat.S_IMODE(os.stat(f1).st_mode)
+        nt.assert_equal(mode, 0o644, '{:o} != 644'.format(mode))
+
+        os.umask(0o057)
+        f2 = os.path.join(td, '2')
+        with atomic_writing(f2) as f:
+            f.write(u'2')
+        mode = stat.S_IMODE(os.stat(f2).st_mode)
+        nt.assert_equal(mode, 0o620, '{:o} != 620'.format(mode))
+
+
+def test_atomic_writing_newlines():
+    with TemporaryDirectory() as td:
+        path = os.path.join(td, 'testfile')
+        
+        lf = u'a\nb\nc\n'
+        plat = lf.replace(u'\n', os.linesep)
+        crlf = lf.replace(u'\n', u'\r\n')
+        
+        # test default
+        with stdlib_io.open(path, 'w') as f:
+            f.write(lf)
+        with stdlib_io.open(path, 'r', newline='') as f:
+            read = f.read()
+        nt.assert_equal(read, plat)
+        
+        # test newline=LF
+        with stdlib_io.open(path, 'w', newline='\n') as f:
+            f.write(lf)
+        with stdlib_io.open(path, 'r', newline='') as f:
+            read = f.read()
+        nt.assert_equal(read, lf)
+        
+        # test newline=CRLF
+        with atomic_writing(path, newline='\r\n') as f:
+            f.write(lf)
+        with stdlib_io.open(path, 'r', newline='') as f:
+            read = f.read()
+        nt.assert_equal(read, crlf)
+        
+        # test newline=no convert
+        text = u'crlf\r\ncr\rlf\n'
+        with atomic_writing(path, newline='') as f:
+            f.write(text)
+        with stdlib_io.open(path, 'r', newline='') as f:
+            read = f.read()
+        nt.assert_equal(read, text)

jupyter_nbconvert/utils/io.py

@@ -0,0 +1,33 b''
+# coding: utf-8
+"""io-related utilities"""
+
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import codecs
+import sys
+from IPython.utils.py3compat import PY3
+
+
+def unicode_std_stream(stream='stdout'):
+    u"""Get a wrapper to write unicode to stdout/stderr as UTF-8.
+
+    This ignores environment variables and default encodings, to reliably write
+    unicode to stdout or stderr.
+
+    ::
+
+        unicode_std_stream().write(u'ł@e¶ŧ←')
+    """
+    assert stream in ('stdout', 'stderr')
+    stream  = getattr(sys, stream)
+    if PY3:
+        try:
+            stream_b = stream.buffer
+        except AttributeError:
+            # sys.stdout has been replaced - use it directly
+            return stream
+    else:
+        stream_b = stream
+
+    return codecs.getwriter('utf-8')(stream_b)

jupyter_nbconvert/utils/tests/test_io.py

@@ -0,0 +1,50 b''
+# encoding: utf-8
+"""Tests for utils.io"""
+
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import io as stdlib_io
+import sys
+
+import nose.tools as nt
+
+from IPython.testing.decorators import skipif
+from ..io import unicode_std_stream
+from IPython.utils.py3compat import PY3
+
+if PY3:
+    from io import StringIO
+else:
+    from StringIO import StringIO
+
+def test_UnicodeStdStream():
+    # Test wrapping a bytes-level stdout
+    if PY3:
+        stdoutb = stdlib_io.BytesIO()
+        stdout = stdlib_io.TextIOWrapper(stdoutb, encoding='ascii')
+    else:
+        stdout = stdoutb = stdlib_io.BytesIO()
+
+    orig_stdout = sys.stdout
+    sys.stdout = stdout
+    try:
+        sample = u"@łe¶ŧ←"
+        unicode_std_stream().write(sample)
+
+        output = stdoutb.getvalue().decode('utf-8')
+        nt.assert_equal(output, sample)
+        assert not stdout.closed
+    finally:
+        sys.stdout = orig_stdout
+
+@skipif(not PY3, "Not applicable on Python 2")
+def test_UnicodeStdStream_nowrap():
+    # If we replace stdout with a StringIO, it shouldn't get wrapped.
+    orig_stdout = sys.stdout
+    sys.stdout = StringIO()
+    try:
+        nt.assert_is(unicode_std_stream(), sys.stdout)
+        assert not sys.stdout.closed
+    finally:
+        sys.stdout = orig_stdout

IPython/html/services/contents/fileio.py

@@ -11,6 +11,7 b' import errno'
 import io
 import os
 import shutil
+import tempfile
 
 from tornado.web import HTTPError
 
@@ -19,10 +20,91 b' from IPython.html.utils import ('
     to_os_path,
 )
 from IPython import nbformat
-from IPython.utils.io import atomic_writing
 from IPython.utils.py3compat import str_to_unicode
 
 
+def _copy_metadata(src, dst):
+    """Copy the set of metadata we want for atomic_writing.
+    
+    Permission bits and flags. We'd like to copy file ownership as well, but we
+    can't do that.
+    """
+    shutil.copymode(src, dst)
+    st = os.stat(src)
+    if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
+        os.chflags(dst, st.st_flags)
+
+@contextmanager
+def atomic_writing(path, text=True, encoding='utf-8', **kwargs):
+    """Context manager to write to a file only if the entire write is successful.
+    
+    This works by creating a temporary file in the same directory, and renaming
+    it over the old file if the context is exited without an error. If other
+    file names are hard linked to the target file, this relationship will not be
+    preserved.
+    
+    On Windows, there is a small chink in the atomicity: the target file is
+    deleted before renaming the temporary file over it. This appears to be
+    unavoidable.
+    
+    Parameters
+    ----------
+    path : str
+      The target file to write to.
+     
+    text : bool, optional
+      Whether to open the file in text mode (i.e. to write unicode). Default is
+      True.
+    
+    encoding : str, optional
+      The encoding to use for files opened in text mode. Default is UTF-8.
+     
+    **kwargs
+      Passed to :func:`io.open`.
+    """
+    # realpath doesn't work on Windows: http://bugs.python.org/issue9949
+    # Luckily, we only need to resolve the file itself being a symlink, not
+    # any of its directories, so this will suffice:
+    if os.path.islink(path):
+        path = os.path.join(os.path.dirname(path), os.readlink(path))
+
+    dirname, basename = os.path.split(path)
+    tmp_dir = tempfile.mkdtemp(prefix=basename, dir=dirname)
+    tmp_path = os.path.join(tmp_dir, basename)
+    if text:
+        fileobj = io.open(tmp_path, 'w', encoding=encoding, **kwargs)
+    else:
+        fileobj = io.open(tmp_path, 'wb', **kwargs)
+
+    try:
+        yield fileobj
+    except:
+        fileobj.close()
+        shutil.rmtree(tmp_dir)
+        raise
+
+    # Flush to disk
+    fileobj.flush()
+    os.fsync(fileobj.fileno())
+
+    # Written successfully, now rename it
+    fileobj.close()
+
+    # Copy permission bits, access time, etc.
+    try:
+        _copy_metadata(path, tmp_path)
+    except OSError:
+        # e.g. the file didn't already exist. Ignore any failure to copy metadata
+        pass
+
+    if os.name == 'nt' and os.path.exists(path):
+        # Rename over existing file doesn't work on Windows
+        os.remove(path)
+
+    os.rename(tmp_path, path)
+    shutil.rmtree(tmp_dir)
+
+
 class FileManagerMixin(object):
     """
     Mixin for ContentsAPI classes that interact with the filesystem.

IPython/utils/io.py

@@ -3,33 +3,24 b''
 IO related utilities.
 """
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
 from __future__ import print_function
 from __future__ import absolute_import
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+
 import codecs
 from contextlib import contextmanager
 import io
 import os
 import shutil
-import stat
 import sys
 import tempfile
+import warnings
 from .capture import CapturedIO, capture_output
 from .py3compat import string_types, input, PY3
 
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
 
 class IOStream:
 
@@ -221,87 +212,11 b" def temp_pyfile(src, ext='.py'):"
     f.flush()
     return fname, f
 
-def _copy_metadata(src, dst):
-    """Copy the set of metadata we want for atomic_writing.
-    
-    Permission bits and flags. We'd like to copy file ownership as well, but we
-    can't do that.
-    """
-    shutil.copymode(src, dst)
-    st = os.stat(src)
-    if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
-        os.chflags(dst, st.st_flags)
-
-@contextmanager
-def atomic_writing(path, text=True, encoding='utf-8', **kwargs):
-    """Context manager to write to a file only if the entire write is successful.
-    
-    This works by creating a temporary file in the same directory, and renaming
-    it over the old file if the context is exited without an error. If other
-    file names are hard linked to the target file, this relationship will not be
-    preserved.
-    
-    On Windows, there is a small chink in the atomicity: the target file is
-    deleted before renaming the temporary file over it. This appears to be
-    unavoidable.
-    
-    Parameters
-    ----------
-    path : str
-      The target file to write to.
-     
-    text : bool, optional
-      Whether to open the file in text mode (i.e. to write unicode). Default is
-      True.
-    
-    encoding : str, optional
-      The encoding to use for files opened in text mode. Default is UTF-8.
-     
-    **kwargs
-      Passed to :func:`io.open`.
-    """
-    # realpath doesn't work on Windows: http://bugs.python.org/issue9949
-    # Luckily, we only need to resolve the file itself being a symlink, not
-    # any of its directories, so this will suffice:
-    if os.path.islink(path):
-        path = os.path.join(os.path.dirname(path), os.readlink(path))
-
-    dirname, basename = os.path.split(path)
-    tmp_dir = tempfile.mkdtemp(prefix=basename, dir=dirname)
-    tmp_path = os.path.join(tmp_dir, basename)
-    if text:
-        fileobj = io.open(tmp_path, 'w', encoding=encoding, **kwargs)
-    else:
-        fileobj = io.open(tmp_path, 'wb', **kwargs)
-
-    try:
-        yield fileobj
-    except:
-        fileobj.close()
-        shutil.rmtree(tmp_dir)
-        raise
-
-    # Flush to disk
-    fileobj.flush()
-    os.fsync(fileobj.fileno())
-
-    # Written successfully, now rename it
-    fileobj.close()
-
-    # Copy permission bits, access time, etc.
-    try:
-        _copy_metadata(path, tmp_path)
-    except OSError:
-        # e.g. the file didn't already exist. Ignore any failure to copy metadata
-        pass
-
-    if os.name == 'nt' and os.path.exists(path):
-        # Rename over existing file doesn't work on Windows
-        os.remove(path)
-
-    os.rename(tmp_path, path)
-    shutil.rmtree(tmp_dir)
-
+def atomic_writing(*args, **kwargs):
+    """DEPRECATED: moved to IPython.html.services.contents.fileio"""
+    warn("IPython.utils.io.atomic_writing has moved to IPython.html.services.contents.fileio")
+    from IPython.html.services.contents.fileio import atomic_writing
+    return atomic_writing(*args, **kwargs)
 
 def raw_print(*args, **kw):
     """Raw print to sys.__stdout__, otherwise identical interface to print()."""
@@ -323,25 +238,9 b' def raw_print_err(*args, **kw):'
 rprint = raw_print
 rprinte = raw_print_err
 
-def unicode_std_stream(stream='stdout'):
-    u"""Get a wrapper to write unicode to stdout/stderr as UTF-8.
-
-    This ignores environment variables and default encodings, to reliably write
-    unicode to stdout or stderr.
 
-    ::
-
-        unicode_std_stream().write(u'ł@e¶ŧ←')
-    """
-    assert stream in ('stdout', 'stderr')
-    stream  = getattr(sys, stream)
-    if PY3:
-        try:
-            stream_b = stream.buffer
-        except AttributeError:
-            # sys.stdout has been replaced - use it directly
-            return stream
-    else:
-        stream_b = stream
-
-    return codecs.getwriter('utf-8')(stream_b)
+def unicode_std_stream(stream='stdout'):
+    """DEPRECATED, moved to jupyter_nbconvert.utils.io"""
+    warn("IPython.utils.io.unicode_std_stream has moved to jupyter_nbconvert.utils.io")
+    from jupyter_nbconvert.utils.io import unicode_std_stream
+    return unicode_std_stream(stream)

IPython/utils/tests/test_io.py

@@ -18,9 +18,7 b' import unittest'
 import nose.tools as nt
 
 from IPython.testing.decorators import skipif, skip_win32
-from IPython.utils.io import (Tee, capture_output, unicode_std_stream,
-                              atomic_writing,
-                              )
+from IPython.utils.io import Tee, capture_output
 from IPython.utils.py3compat import doctest_refactor_print, PY3
 from IPython.utils.tempdir import TemporaryDirectory
 
@@ -86,146 +84,4 b' def test_capture_output():'
     nt.assert_equal(io.stdout, 'hi, stdout\n')
     nt.assert_equal(io.stderr, 'hi, stderr\n')
 
-def test_UnicodeStdStream():
-    # Test wrapping a bytes-level stdout
-    if PY3:
-        stdoutb = stdlib_io.BytesIO()
-        stdout = stdlib_io.TextIOWrapper(stdoutb, encoding='ascii')
-    else:
-        stdout = stdoutb = stdlib_io.BytesIO()
-
-    orig_stdout = sys.stdout
-    sys.stdout = stdout
-    try:
-        sample = u"@łe¶ŧ←"
-        unicode_std_stream().write(sample)
-
-        output = stdoutb.getvalue().decode('utf-8')
-        nt.assert_equal(output, sample)
-        assert not stdout.closed
-    finally:
-        sys.stdout = orig_stdout
-
-@skipif(not PY3, "Not applicable on Python 2")
-def test_UnicodeStdStream_nowrap():
-    # If we replace stdout with a StringIO, it shouldn't get wrapped.
-    orig_stdout = sys.stdout
-    sys.stdout = StringIO()
-    try:
-        nt.assert_is(unicode_std_stream(), sys.stdout)
-        assert not sys.stdout.closed
-    finally:
-        sys.stdout = orig_stdout
-
-def test_atomic_writing():
-    class CustomExc(Exception): pass
-
-    with TemporaryDirectory() as td:
-        f1 = os.path.join(td, 'penguin')
-        with stdlib_io.open(f1, 'w') as f:
-            f.write(u'Before')
-        
-        if os.name != 'nt':
-            os.chmod(f1, 0o701)
-            orig_mode = stat.S_IMODE(os.stat(f1).st_mode)
-
-        f2 = os.path.join(td, 'flamingo')
-        try:
-            os.symlink(f1, f2)
-            have_symlink = True
-        except (AttributeError, NotImplementedError, OSError):
-            # AttributeError: Python doesn't support it
-            # NotImplementedError: The system doesn't support it
-            # OSError: The user lacks the privilege (Windows)
-            have_symlink = False
-
-        with nt.assert_raises(CustomExc):
-            with atomic_writing(f1) as f:
-                f.write(u'Failing write')
-                raise CustomExc
-
-        # Because of the exception, the file should not have been modified
-        with stdlib_io.open(f1, 'r') as f:
-            nt.assert_equal(f.read(), u'Before')
-
-        with atomic_writing(f1) as f:
-            f.write(u'Overwritten')
-
-        with stdlib_io.open(f1, 'r') as f:
-            nt.assert_equal(f.read(), u'Overwritten')
-
-        if os.name != 'nt':
-            mode = stat.S_IMODE(os.stat(f1).st_mode)
-            nt.assert_equal(mode, orig_mode)
-
-        if have_symlink:
-            # Check that writing over a file preserves a symlink
-            with atomic_writing(f2) as f:
-                f.write(u'written from symlink')
-            
-            with stdlib_io.open(f1, 'r') as f:
-                nt.assert_equal(f.read(), u'written from symlink')
-
-def _save_umask():
-    global umask
-    umask = os.umask(0)
-    os.umask(umask)
-
-def _restore_umask():
-    os.umask(umask)
-
-@skip_win32
[e-mail]_setup(_save_umask, _restore_umask)
-def test_atomic_writing_umask():
-    with TemporaryDirectory() as td:
-        os.umask(0o022)
-        f1 = os.path.join(td, '1')
-        with atomic_writing(f1) as f:
-            f.write(u'1')
-        mode = stat.S_IMODE(os.stat(f1).st_mode)
-        nt.assert_equal(mode, 0o644, '{:o} != 644'.format(mode))
-
-        os.umask(0o057)
-        f2 = os.path.join(td, '2')
-        with atomic_writing(f2) as f:
-            f.write(u'2')
-        mode = stat.S_IMODE(os.stat(f2).st_mode)
-        nt.assert_equal(mode, 0o620, '{:o} != 620'.format(mode))
-
-
-def test_atomic_writing_newlines():
-    with TemporaryDirectory() as td:
-        path = os.path.join(td, 'testfile')
-        
-        lf = u'a\nb\nc\n'
-        plat = lf.replace(u'\n', os.linesep)
-        crlf = lf.replace(u'\n', u'\r\n')
-        
-        # test default
-        with stdlib_io.open(path, 'w') as f:
-            f.write(lf)
-        with stdlib_io.open(path, 'r', newline='') as f:
-            read = f.read()
-        nt.assert_equal(read, plat)
-        
-        # test newline=LF
-        with stdlib_io.open(path, 'w', newline='\n') as f:
-            f.write(lf)
-        with stdlib_io.open(path, 'r', newline='') as f:
-            read = f.read()
-        nt.assert_equal(read, lf)
-        
-        # test newline=CRLF
-        with atomic_writing(path, newline='\r\n') as f:
-            f.write(lf)
-        with stdlib_io.open(path, 'r', newline='') as f:
-            read = f.read()
-        nt.assert_equal(read, crlf)
-        
-        # test newline=no convert
-        text = u'crlf\r\ncr\rlf\n'
-        with atomic_writing(path, newline='') as f:
-            f.write(text)
-        with stdlib_io.open(path, 'r', newline='') as f:
-            read = f.read()
-        nt.assert_equal(read, text)
+

jupyter_client/session.py

@@ -49,7 +49,6 b' from zmq.eventloop.zmqstream import ZMQStream'
 
 from IPython.core.release import kernel_protocol_version
 from IPython.config.configurable import Configurable, LoggingConfigurable
-from IPython.utils import io
 from IPython.utils.importstring import import_item
 from jupyter_client.jsonutil import extract_dates, squash_dates, date_default
 from IPython.utils.py3compat import (str_to_bytes, str_to_unicode, unicode_type,
@@ -59,6 +58,8 b' from IPython.utils.traitlets import (CBytes, Unicode, Bool, Any, Instance, Set,'
                                         TraitError,
 )
 from jupyter_client.adapter import adapt
+from traitlets.log import get_logger
+
 
 #-----------------------------------------------------------------------------
 # utility functions
@@ -653,8 +654,9 b' class Session(Configurable):'
             msg = self.msg(msg_or_type, content=content, parent=parent,
                            header=header, metadata=metadata)
         if not os.getpid() == self.pid:
-            io.rprint("WARNING: attempted to send message from fork")
-            io.rprint(msg)
+            get_logger().warn("WARNING: attempted to send message from fork\n%s",
+                msg
+            )
             return
         buffers = [] if buffers is None else buffers
         if self.adapt_version:

jupyter_nbconvert/utils/pandoc.py

@@ -1,33 +1,20 b''
 """Utility for calling pandoc"""
-#-----------------------------------------------------------------------------
-# Copyright (c) 2014 the IPython Development Team.
-#
+# Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-from __future__ import print_function
+from __future__ import print_function, absolute_import
 
-# Stdlib imports
 import subprocess
 import warnings
 import re
 from io import TextIOWrapper, BytesIO
 
-# IPython imports
 from IPython.utils.py3compat import cast_bytes
 from IPython.utils.version import check_version
 from IPython.utils.process import is_cmd_found, FindCmdError
 
 from .exceptions import ConversionException
 
-#-----------------------------------------------------------------------------
-# Classes and functions
-#-----------------------------------------------------------------------------
 _minimal_version = "1.12.1"
 
 def pandoc(source, fmt, to, extra_args=None, encoding='utf-8'):

jupyter_nbconvert/writers/stdout.py

@@ -1,24 +1,13 b''
 """
 Contains Stdout writer
 """
-#-----------------------------------------------------------------------------
-#Copyright (c) 2013, the IPython Development Team.
-#
-#Distributed under the terms of the Modified BSD License.
-#
-#The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
 
-from IPython.utils import io
+from jupyter_nbconvert.utils import io
 from .base import WriterBase
 
-#-----------------------------------------------------------------------------
-# Classes
-#-----------------------------------------------------------------------------
 
 class StdoutWriter(WriterBase):
     """Consumes output from nbconvert export...() methods and writes to the