Merge pull request #8240 from minrk/split-io

split utils.io

IPython/html/services/contents/tests/test_fileio.py

@@ -0,0 +1,131 b''
+# encoding: utf-8
+"""Tests for file IO"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import io as stdlib_io
+import os.path
+import stat
+
+import nose.tools as nt
+
+from IPython.testing.decorators import skip_win32
+from ..fileio import atomic_writing
+
+from IPython.utils.tempdir import TemporaryDirectory
+
+umask = 0
+
+def test_atomic_writing():
+    class CustomExc(Exception): pass
+
+    with TemporaryDirectory() as td:
+        f1 = os.path.join(td, 'penguin')
+        with stdlib_io.open(f1, 'w') as f:
+            f.write(u'Before')
+        
+        if os.name != 'nt':
+            os.chmod(f1, 0o701)
+            orig_mode = stat.S_IMODE(os.stat(f1).st_mode)
+
+        f2 = os.path.join(td, 'flamingo')
+        try:
+            os.symlink(f1, f2)
+            have_symlink = True
+        except (AttributeError, NotImplementedError, OSError):
+            # AttributeError: Python doesn't support it
+            # NotImplementedError: The system doesn't support it
+            # OSError: The user lacks the privilege (Windows)
+            have_symlink = False
+
+        with nt.assert_raises(CustomExc):
+            with atomic_writing(f1) as f:
+                f.write(u'Failing write')
+                raise CustomExc
+
+        # Because of the exception, the file should not have been modified
+        with stdlib_io.open(f1, 'r') as f:
+            nt.assert_equal(f.read(), u'Before')
+
+        with atomic_writing(f1) as f:
+            f.write(u'Overwritten')
+
+        with stdlib_io.open(f1, 'r') as f:
+            nt.assert_equal(f.read(), u'Overwritten')
+
+        if os.name != 'nt':
+            mode = stat.S_IMODE(os.stat(f1).st_mode)
+            nt.assert_equal(mode, orig_mode)
+
+        if have_symlink:
+            # Check that writing over a file preserves a symlink
+            with atomic_writing(f2) as f:
+                f.write(u'written from symlink')
+            
+            with stdlib_io.open(f1, 'r') as f:
+                nt.assert_equal(f.read(), u'written from symlink')
+
+def _save_umask():
+    global umask
+    umask = os.umask(0)
+    os.umask(umask)
+
+def _restore_umask():
+    os.umask(umask)
+
+@skip_win32
[e-mail]_setup(_save_umask, _restore_umask)
+def test_atomic_writing_umask():
+    with TemporaryDirectory() as td:
+        os.umask(0o022)
+        f1 = os.path.join(td, '1')
+        with atomic_writing(f1) as f:
+            f.write(u'1')
+        mode = stat.S_IMODE(os.stat(f1).st_mode)
+        nt.assert_equal(mode, 0o644, '{:o} != 644'.format(mode))
+
+        os.umask(0o057)
+        f2 = os.path.join(td, '2')
+        with atomic_writing(f2) as f:
+            f.write(u'2')
+        mode = stat.S_IMODE(os.stat(f2).st_mode)
+        nt.assert_equal(mode, 0o620, '{:o} != 620'.format(mode))
+
+
+def test_atomic_writing_newlines():
+    with TemporaryDirectory() as td:
+        path = os.path.join(td, 'testfile')
+        
+        lf = u'a\nb\nc\n'
+        plat = lf.replace(u'\n', os.linesep)
+        crlf = lf.replace(u'\n', u'\r\n')
+        
+        # test default
+        with stdlib_io.open(path, 'w') as f:
+            f.write(lf)
+        with stdlib_io.open(path, 'r', newline='') as f:
+            read = f.read()
+        nt.assert_equal(read, plat)
+        
+        # test newline=LF
+        with stdlib_io.open(path, 'w', newline='\n') as f:
+            f.write(lf)
+        with stdlib_io.open(path, 'r', newline='') as f:
+            read = f.read()
+        nt.assert_equal(read, lf)
+        
+        # test newline=CRLF
+        with atomic_writing(path, newline='\r\n') as f:
+            f.write(lf)
+        with stdlib_io.open(path, 'r', newline='') as f:
+            read = f.read()
+        nt.assert_equal(read, crlf)
+        
+        # test newline=no convert
+        text = u'crlf\r\ncr\rlf\n'
+        with atomic_writing(path, newline='') as f:
+            f.write(text)
+        with stdlib_io.open(path, 'r', newline='') as f:
+            read = f.read()
+        nt.assert_equal(read, text)

jupyter_nbconvert/utils/io.py

@@ -0,0 +1,33 b''
+# coding: utf-8
+"""io-related utilities"""
+
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import codecs
+import sys
+from IPython.utils.py3compat import PY3
+
+
+def unicode_std_stream(stream='stdout'):
+    u"""Get a wrapper to write unicode to stdout/stderr as UTF-8.
+
+    This ignores environment variables and default encodings, to reliably write
+    unicode to stdout or stderr.
+
+    ::
+
+        unicode_std_stream().write(u'ł@e¶ŧ←')
+    """
+    assert stream in ('stdout', 'stderr')
+    stream  = getattr(sys, stream)
+    if PY3:
+        try:
+            stream_b = stream.buffer
+        except AttributeError:
+            # sys.stdout has been replaced - use it directly
+            return stream
+    else:
+        stream_b = stream
+
+    return codecs.getwriter('utf-8')(stream_b)

jupyter_nbconvert/utils/tests/test_io.py

@@ -0,0 +1,50 b''
+# encoding: utf-8
+"""Tests for utils.io"""
+
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import io as stdlib_io
+import sys
+
+import nose.tools as nt
+
+from IPython.testing.decorators import skipif
+from ..io import unicode_std_stream
+from IPython.utils.py3compat import PY3
+
+if PY3:
+    from io import StringIO
+else:
+    from StringIO import StringIO
+
+def test_UnicodeStdStream():
+    # Test wrapping a bytes-level stdout
+    if PY3:
+        stdoutb = stdlib_io.BytesIO()
+        stdout = stdlib_io.TextIOWrapper(stdoutb, encoding='ascii')
+    else:
+        stdout = stdoutb = stdlib_io.BytesIO()
+
+    orig_stdout = sys.stdout
+    sys.stdout = stdout
+    try:
+        sample = u"@łe¶ŧ←"
+        unicode_std_stream().write(sample)
+
+        output = stdoutb.getvalue().decode('utf-8')
+        nt.assert_equal(output, sample)
+        assert not stdout.closed
+    finally:
+        sys.stdout = orig_stdout
+
+@skipif(not PY3, "Not applicable on Python 2")
+def test_UnicodeStdStream_nowrap():
+    # If we replace stdout with a StringIO, it shouldn't get wrapped.
+    orig_stdout = sys.stdout
+    sys.stdout = StringIO()
+    try:
+        nt.assert_is(unicode_std_stream(), sys.stdout)
+        assert not sys.stdout.closed
+    finally:
+        sys.stdout = orig_stdout

IPython/html/services/contents/fileio.py

@@ -1,174 +1,256 b''
 """
 Utilities for file-based Contents/Checkpoints managers.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import base64
 from contextlib import contextmanager
 import errno
 import io
 import os
 import shutil
+import tempfile
 
 from tornado.web import HTTPError
 
 from IPython.html.utils import (
     to_api_path,
     to_os_path,
 )
 from IPython import nbformat
-from IPython.utils.io import atomic_writing
 from IPython.utils.py3compat import str_to_unicode
 
 
+def _copy_metadata(src, dst):
+    """Copy the set of metadata we want for atomic_writing.
+    
+    Permission bits and flags. We'd like to copy file ownership as well, but we
+    can't do that.
+    """
+    shutil.copymode(src, dst)
+    st = os.stat(src)
+    if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
+        os.chflags(dst, st.st_flags)
+
+@contextmanager
+def atomic_writing(path, text=True, encoding='utf-8', **kwargs):
+    """Context manager to write to a file only if the entire write is successful.
+    
+    This works by creating a temporary file in the same directory, and renaming
+    it over the old file if the context is exited without an error. If other
+    file names are hard linked to the target file, this relationship will not be
+    preserved.
+    
+    On Windows, there is a small chink in the atomicity: the target file is
+    deleted before renaming the temporary file over it. This appears to be
+    unavoidable.
+    
+    Parameters
+    ----------
+    path : str
+      The target file to write to.
+     
+    text : bool, optional
+      Whether to open the file in text mode (i.e. to write unicode). Default is
+      True.
+    
+    encoding : str, optional
+      The encoding to use for files opened in text mode. Default is UTF-8.
+     
+    **kwargs
+      Passed to :func:`io.open`.
+    """
+    # realpath doesn't work on Windows: http://bugs.python.org/issue9949
+    # Luckily, we only need to resolve the file itself being a symlink, not
+    # any of its directories, so this will suffice:
+    if os.path.islink(path):
+        path = os.path.join(os.path.dirname(path), os.readlink(path))
+
+    dirname, basename = os.path.split(path)
+    tmp_dir = tempfile.mkdtemp(prefix=basename, dir=dirname)
+    tmp_path = os.path.join(tmp_dir, basename)
+    if text:
+        fileobj = io.open(tmp_path, 'w', encoding=encoding, **kwargs)
+    else:
+        fileobj = io.open(tmp_path, 'wb', **kwargs)
+
+    try:
+        yield fileobj
+    except:
+        fileobj.close()
+        shutil.rmtree(tmp_dir)
+        raise
+
+    # Flush to disk
+    fileobj.flush()
+    os.fsync(fileobj.fileno())
+
+    # Written successfully, now rename it
+    fileobj.close()
+
+    # Copy permission bits, access time, etc.
+    try:
+        _copy_metadata(path, tmp_path)
+    except OSError:
+        # e.g. the file didn't already exist. Ignore any failure to copy metadata
+        pass
+
+    if os.name == 'nt' and os.path.exists(path):
+        # Rename over existing file doesn't work on Windows
+        os.remove(path)
+
+    os.rename(tmp_path, path)
+    shutil.rmtree(tmp_dir)
+
+
 class FileManagerMixin(object):
     """
     Mixin for ContentsAPI classes that interact with the filesystem.
 
     Provides facilities for reading, writing, and copying both notebooks and
     generic files.
 
     Shared by FileContentsManager and FileCheckpoints.
 
     Note
     ----
     Classes using this mixin must provide the following attributes:
 
     root_dir : unicode
         A directory against against which API-style paths are to be resolved.
 
     log : logging.Logger
     """
 
     @contextmanager
     def open(self, os_path, *args, **kwargs):
         """wrapper around io.open that turns permission errors into 403"""
         with self.perm_to_403(os_path):
             with io.open(os_path, *args, **kwargs) as f:
                 yield f
 
     @contextmanager
     def atomic_writing(self, os_path, *args, **kwargs):
         """wrapper around atomic_writing that turns permission errors to 403"""
         with self.perm_to_403(os_path):
             with atomic_writing(os_path, *args, **kwargs) as f:
                 yield f
 
     @contextmanager
     def perm_to_403(self, os_path=''):
         """context manager for turning permission errors into 403."""
         try:
             yield
         except (OSError, IOError) as e:
             if e.errno in {errno.EPERM, errno.EACCES}:
                 # make 403 error message without root prefix
                 # this may not work perfectly on unicode paths on Python 2,
                 # but nobody should be doing that anyway.
                 if not os_path:
                     os_path = str_to_unicode(e.filename or 'unknown file')
                 path = to_api_path(os_path, root=self.root_dir)
                 raise HTTPError(403, u'Permission denied: %s' % path)
             else:
                 raise
 
     def _copy(self, src, dest):
         """copy src to dest
 
         like shutil.copy2, but log errors in copystat
         """
         shutil.copyfile(src, dest)
         try:
             shutil.copystat(src, dest)
         except OSError:
             self.log.debug("copystat on %s failed", dest, exc_info=True)
 
     def _get_os_path(self, path):
         """Given an API path, return its file system path.
 
         Parameters
         ----------
         path : string
             The relative API path to the named file.
 
         Returns
         -------
         path : string
             Native, absolute OS path to for a file.
 
         Raises
         ------
         404: if path is outside root
         """
         root = os.path.abspath(self.root_dir)
         os_path = to_os_path(path, root)
         if not (os.path.abspath(os_path) + os.path.sep).startswith(root):
             raise HTTPError(404, "%s is outside root contents directory" % path)
         return os_path
 
     def _read_notebook(self, os_path, as_version=4):
         """Read a notebook from an os path."""
         with self.open(os_path, 'r', encoding='utf-8') as f:
             try:
                 return nbformat.read(f, as_version=as_version)
             except Exception as e:
                 raise HTTPError(
                     400,
                     u"Unreadable Notebook: %s %r" % (os_path, e),
                 )
 
     def _save_notebook(self, os_path, nb):
         """Save a notebook to an os_path."""
         with self.atomic_writing(os_path, encoding='utf-8') as f:
             nbformat.write(nb, f, version=nbformat.NO_CONVERT)
 
     def _read_file(self, os_path, format):
         """Read a non-notebook file.
 
         os_path: The path to be read.
         format:
           If 'text', the contents will be decoded as UTF-8.
           If 'base64', the raw bytes contents will be encoded as base64.
           If not specified, try to decode as UTF-8, and fall back to base64
         """
         if not os.path.isfile(os_path):
             raise HTTPError(400, "Cannot read non-file %s" % os_path)
 
         with self.open(os_path, 'rb') as f:
             bcontent = f.read()
 
         if format is None or format == 'text':
             # Try to interpret as unicode if format is unknown or if unicode
             # was explicitly requested.
             try:
                 return bcontent.decode('utf8'), 'text'
             except UnicodeError:
                 if format == 'text':
                     raise HTTPError(
                         400,
                         "%s is not UTF-8 encoded" % os_path,
                         reason='bad format',
                     )
         return base64.encodestring(bcontent).decode('ascii'), 'base64'
 
     def _save_file(self, os_path, content, format):
         """Save content of a generic file."""
         if format not in {'text', 'base64'}:
             raise HTTPError(
                 400,
                 "Must specify format of file contents as 'text' or 'base64'",
             )
         try:
             if format == 'text':
                 bcontent = content.encode('utf8')
             else:
                 b64_bytes = content.encode('ascii')
                 bcontent = base64.decodestring(b64_bytes)
         except Exception as e:
             raise HTTPError(
                 400, u'Encoding error saving %s: %s' % (os_path, e)
             )
 
         with self.atomic_writing(os_path, text=False) as f:
             f.write(bcontent)

IPython/utils/io.py

@@ -1,347 +1,246 b''
 # encoding: utf-8
 """
 IO related utilities.
 """
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
 from __future__ import print_function
 from __future__ import absolute_import
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+
 import codecs
 from contextlib import contextmanager
 import io
 import os
 import shutil
-import stat
 import sys
 import tempfile
+import warnings
 from .capture import CapturedIO, capture_output
 from .py3compat import string_types, input, PY3
 
-#-----------------------------------------------------------------------------
-# Code
-#-----------------------------------------------------------------------------
-
 
 class IOStream:
 
     def __init__(self,stream, fallback=None):
         if not hasattr(stream,'write') or not hasattr(stream,'flush'):
             if fallback is not None:
                 stream = fallback
             else:
                 raise ValueError("fallback required, but not specified")
         self.stream = stream
         self._swrite = stream.write
 
         # clone all methods not overridden:
         def clone(meth):
             return not hasattr(self, meth) and not meth.startswith('_')
         for meth in filter(clone, dir(stream)):
             setattr(self, meth, getattr(stream, meth))
 
     def __repr__(self):
         cls = self.__class__
         tpl = '{mod}.{cls}({args})'
         return tpl.format(mod=cls.__module__, cls=cls.__name__, args=self.stream)
 
     def write(self,data):
         try:
             self._swrite(data)
         except:
             try:
                 # print handles some unicode issues which may trip a plain
                 # write() call.  Emulate write() by using an empty end
                 # argument.
                 print(data, end='', file=self.stream)
             except:
                 # if we get here, something is seriously broken.
                 print('ERROR - failed to write data to stream:', self.stream,
                       file=sys.stderr)
 
     def writelines(self, lines):
         if isinstance(lines, string_types):
             lines = [lines]
         for line in lines:
             self.write(line)
 
     # This class used to have a writeln method, but regular files and streams
     # in Python don't have this method. We need to keep this completely
     # compatible so we removed it.
 
     @property
     def closed(self):
         return self.stream.closed
 
     def close(self):
         pass
 
 # setup stdin/stdout/stderr to sys.stdin/sys.stdout/sys.stderr
 devnull = open(os.devnull, 'w') 
 stdin = IOStream(sys.stdin, fallback=devnull)
 stdout = IOStream(sys.stdout, fallback=devnull)
 stderr = IOStream(sys.stderr, fallback=devnull)
 
 class IOTerm:
     """ Term holds the file or file-like objects for handling I/O operations.
 
     These are normally just sys.stdin, sys.stdout and sys.stderr but for
     Windows they can can replaced to allow editing the strings before they are
     displayed."""
 
     # In the future, having IPython channel all its I/O operations through
     # this class will make it easier to embed it into other environments which
     # are not a normal terminal (such as a GUI-based shell)
     def __init__(self, stdin=None, stdout=None, stderr=None):
         mymodule = sys.modules[__name__]
         self.stdin  = IOStream(stdin, mymodule.stdin)
         self.stdout = IOStream(stdout, mymodule.stdout)
         self.stderr = IOStream(stderr, mymodule.stderr)
 
 
 class Tee(object):
     """A class to duplicate an output stream to stdout/err.
 
     This works in a manner very similar to the Unix 'tee' command.
 
     When the object is closed or deleted, it closes the original file given to
     it for duplication.
     """
     # Inspired by:
     # http://mail.python.org/pipermail/python-list/2007-May/442737.html
 
     def __init__(self, file_or_name, mode="w", channel='stdout'):
         """Construct a new Tee object.
 
         Parameters
         ----------
         file_or_name : filename or open filehandle (writable)
           File that will be duplicated
 
         mode : optional, valid mode for open().
           If a filename was give, open with this mode.
 
         channel : str, one of ['stdout', 'stderr']
         """
         if channel not in ['stdout', 'stderr']:
             raise ValueError('Invalid channel spec %s' % channel)
 
         if hasattr(file_or_name, 'write') and hasattr(file_or_name, 'seek'):
             self.file = file_or_name
         else:
             self.file = open(file_or_name, mode)
         self.channel = channel
         self.ostream = getattr(sys, channel)
         setattr(sys, channel, self)
         self._closed = False
 
     def close(self):
         """Close the file and restore the channel."""
         self.flush()
         setattr(sys, self.channel, self.ostream)
         self.file.close()
         self._closed = True
 
     def write(self, data):
         """Write data to both channels."""
         self.file.write(data)
         self.ostream.write(data)
         self.ostream.flush()
 
     def flush(self):
         """Flush both channels."""
         self.file.flush()
         self.ostream.flush()
 
     def __del__(self):
         if not self._closed:
             self.close()
 
 
 def ask_yes_no(prompt, default=None, interrupt=None):
     """Asks a question and returns a boolean (y/n) answer.
 
     If default is given (one of 'y','n'), it is used if the user input is
     empty. If interrupt is given (one of 'y','n'), it is used if the user
     presses Ctrl-C. Otherwise the question is repeated until an answer is
     given.
 
     An EOF is treated as the default answer.  If there is no default, an
     exception is raised to prevent infinite loops.
 
     Valid answers are: y/yes/n/no (match is not case sensitive)."""
 
     answers = {'y':True,'n':False,'yes':True,'no':False}
     ans = None
     while ans not in answers.keys():
         try:
             ans = input(prompt+' ').lower()
             if not ans:  # response was an empty string
                 ans = default
         except KeyboardInterrupt:
             if interrupt:
                 ans = interrupt
         except EOFError:
             if default in answers.keys():
                 ans = default
                 print()
             else:
                 raise
 
     return answers[ans]
 
 
 def temp_pyfile(src, ext='.py'):
     """Make a temporary python file, return filename and filehandle.
 
     Parameters
     ----------
     src : string or list of strings (no need for ending newlines if list)
       Source code to be written to the file.
 
     ext : optional, string
       Extension for the generated file.
 
     Returns
     -------
     (filename, open filehandle)
       It is the caller's responsibility to close the open file and unlink it.
     """
     fname = tempfile.mkstemp(ext)[1]
     f = open(fname,'w')
     f.write(src)
     f.flush()
     return fname, f
 
-def _copy_metadata(src, dst):
-    """Copy the set of metadata we want for atomic_writing.
-    
-    Permission bits and flags. We'd like to copy file ownership as well, but we
-    can't do that.
-    """
-    shutil.copymode(src, dst)
-    st = os.stat(src)
-    if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
-        os.chflags(dst, st.st_flags)
-
-@contextmanager
-def atomic_writing(path, text=True, encoding='utf-8', **kwargs):
-    """Context manager to write to a file only if the entire write is successful.
-    
-    This works by creating a temporary file in the same directory, and renaming
-    it over the old file if the context is exited without an error. If other
-    file names are hard linked to the target file, this relationship will not be
-    preserved.
-    
-    On Windows, there is a small chink in the atomicity: the target file is
-    deleted before renaming the temporary file over it. This appears to be
-    unavoidable.
-    
-    Parameters
-    ----------
-    path : str
-      The target file to write to.
-     
-    text : bool, optional
-      Whether to open the file in text mode (i.e. to write unicode). Default is
-      True.
-    
-    encoding : str, optional
-      The encoding to use for files opened in text mode. Default is UTF-8.
-     
-    **kwargs
-      Passed to :func:`io.open`.
-    """
-    # realpath doesn't work on Windows: http://bugs.python.org/issue9949
-    # Luckily, we only need to resolve the file itself being a symlink, not
-    # any of its directories, so this will suffice:
-    if os.path.islink(path):
-        path = os.path.join(os.path.dirname(path), os.readlink(path))
-
-    dirname, basename = os.path.split(path)
-    tmp_dir = tempfile.mkdtemp(prefix=basename, dir=dirname)
-    tmp_path = os.path.join(tmp_dir, basename)
-    if text:
-        fileobj = io.open(tmp_path, 'w', encoding=encoding, **kwargs)
-    else:
-        fileobj = io.open(tmp_path, 'wb', **kwargs)
-
-    try:
-        yield fileobj
-    except:
-        fileobj.close()
-        shutil.rmtree(tmp_dir)
-        raise
-
-    # Flush to disk
-    fileobj.flush()
-    os.fsync(fileobj.fileno())
-
-    # Written successfully, now rename it
-    fileobj.close()
-
-    # Copy permission bits, access time, etc.
-    try:
-        _copy_metadata(path, tmp_path)
-    except OSError:
-        # e.g. the file didn't already exist. Ignore any failure to copy metadata
-        pass
-
-    if os.name == 'nt' and os.path.exists(path):
-        # Rename over existing file doesn't work on Windows
-        os.remove(path)
-
-    os.rename(tmp_path, path)
-    shutil.rmtree(tmp_dir)
-
+def atomic_writing(*args, **kwargs):
+    """DEPRECATED: moved to IPython.html.services.contents.fileio"""
+    warn("IPython.utils.io.atomic_writing has moved to IPython.html.services.contents.fileio")
+    from IPython.html.services.contents.fileio import atomic_writing
+    return atomic_writing(*args, **kwargs)
 
 def raw_print(*args, **kw):
     """Raw print to sys.__stdout__, otherwise identical interface to print()."""
 
     print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
           file=sys.__stdout__)
     sys.__stdout__.flush()
 
 
 def raw_print_err(*args, **kw):
     """Raw print to sys.__stderr__, otherwise identical interface to print()."""
 
     print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
           file=sys.__stderr__)
     sys.__stderr__.flush()
 
 
 # Short aliases for quick debugging, do NOT use these in production code.
 rprint = raw_print
 rprinte = raw_print_err
 
-def unicode_std_stream(stream='stdout'):
-    u"""Get a wrapper to write unicode to stdout/stderr as UTF-8.
-
-    This ignores environment variables and default encodings, to reliably write
-    unicode to stdout or stderr.
 
-    ::
-
-        unicode_std_stream().write(u'ł@e¶ŧ←')
-    """
-    assert stream in ('stdout', 'stderr')
-    stream  = getattr(sys, stream)
-    if PY3:
-        try:
-            stream_b = stream.buffer
-        except AttributeError:
-            # sys.stdout has been replaced - use it directly
-            return stream
-    else:
-        stream_b = stream
-
-    return codecs.getwriter('utf-8')(stream_b)
+def unicode_std_stream(stream='stdout'):
+    """DEPRECATED, moved to jupyter_nbconvert.utils.io"""
+    warn("IPython.utils.io.unicode_std_stream has moved to jupyter_nbconvert.utils.io")
+    from jupyter_nbconvert.utils.io import unicode_std_stream
+    return unicode_std_stream(stream)

IPython/utils/tests/test_io.py

@@ -1,231 +1,87 b''
 # encoding: utf-8
 """Tests for io.py"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 from __future__ import absolute_import
 
 import io as stdlib_io
 import os.path
 import stat
 import sys
 
 from subprocess import Popen, PIPE
 import unittest
 
 import nose.tools as nt
 
 from IPython.testing.decorators import skipif, skip_win32
-from IPython.utils.io import (Tee, capture_output, unicode_std_stream,
-                              atomic_writing,
-                              )
+from IPython.utils.io import Tee, capture_output
 from IPython.utils.py3compat import doctest_refactor_print, PY3
 from IPython.utils.tempdir import TemporaryDirectory
 
 if PY3:
     from io import StringIO
 else:
     from StringIO import StringIO
 
 
 def test_tee_simple():
     "Very simple check with stdout only"
     chan = StringIO()
     text = 'Hello'
     tee = Tee(chan, channel='stdout')
     print(text, file=chan)
     nt.assert_equal(chan.getvalue(), text+"\n")
 
 
 class TeeTestCase(unittest.TestCase):
 
     def tchan(self, channel, check='close'):
         trap = StringIO()
         chan = StringIO()
         text = 'Hello'
         
         std_ori = getattr(sys, channel)
         setattr(sys, channel, trap)
 
         tee = Tee(chan, channel=channel)
         print(text, end='', file=chan)
         setattr(sys, channel, std_ori)
         trap_val = trap.getvalue()
         nt.assert_equal(chan.getvalue(), text)
         if check=='close':
             tee.close()
         else:
             del tee
 
     def test(self):
         for chan in ['stdout', 'stderr']:
             for check in ['close', 'del']:
                 self.tchan(chan, check)
 
 def test_io_init():
     """Test that io.stdin/out/err exist at startup"""
     for name in ('stdin', 'stdout', 'stderr'):
         cmd = doctest_refactor_print("from IPython.utils import io;print io.%s.__class__"%name)
         p = Popen([sys.executable, '-c', cmd],
                     stdout=PIPE)
         p.wait()
         classname = p.stdout.read().strip().decode('ascii')
         # __class__ is a reference to the class object in Python 3, so we can't
         # just test for string equality.
         assert 'IPython.utils.io.IOStream' in classname, classname
 
 def test_capture_output():
     """capture_output() context works"""
     
     with capture_output() as io:
         print('hi, stdout')
         print('hi, stderr', file=sys.stderr)
     
     nt.assert_equal(io.stdout, 'hi, stdout\n')
     nt.assert_equal(io.stderr, 'hi, stderr\n')
 
-def test_UnicodeStdStream():
-    # Test wrapping a bytes-level stdout
-    if PY3:
-        stdoutb = stdlib_io.BytesIO()
-        stdout = stdlib_io.TextIOWrapper(stdoutb, encoding='ascii')
-    else:
-        stdout = stdoutb = stdlib_io.BytesIO()
-
-    orig_stdout = sys.stdout
-    sys.stdout = stdout
-    try:
-        sample = u"@łe¶ŧ←"
-        unicode_std_stream().write(sample)
-
-        output = stdoutb.getvalue().decode('utf-8')
-        nt.assert_equal(output, sample)
-        assert not stdout.closed
-    finally:
-        sys.stdout = orig_stdout
-
-@skipif(not PY3, "Not applicable on Python 2")
-def test_UnicodeStdStream_nowrap():
-    # If we replace stdout with a StringIO, it shouldn't get wrapped.
-    orig_stdout = sys.stdout
-    sys.stdout = StringIO()
-    try:
-        nt.assert_is(unicode_std_stream(), sys.stdout)
-        assert not sys.stdout.closed
-    finally:
-        sys.stdout = orig_stdout
-
-def test_atomic_writing():
-    class CustomExc(Exception): pass
-
-    with TemporaryDirectory() as td:
-        f1 = os.path.join(td, 'penguin')
-        with stdlib_io.open(f1, 'w') as f:
-            f.write(u'Before')
-        
-        if os.name != 'nt':
-            os.chmod(f1, 0o701)
-            orig_mode = stat.S_IMODE(os.stat(f1).st_mode)
-
-        f2 = os.path.join(td, 'flamingo')
-        try:
-            os.symlink(f1, f2)
-            have_symlink = True
-        except (AttributeError, NotImplementedError, OSError):
-            # AttributeError: Python doesn't support it
-            # NotImplementedError: The system doesn't support it
-            # OSError: The user lacks the privilege (Windows)
-            have_symlink = False
-
-        with nt.assert_raises(CustomExc):
-            with atomic_writing(f1) as f:
-                f.write(u'Failing write')
-                raise CustomExc
-
-        # Because of the exception, the file should not have been modified
-        with stdlib_io.open(f1, 'r') as f:
-            nt.assert_equal(f.read(), u'Before')
-
-        with atomic_writing(f1) as f:
-            f.write(u'Overwritten')
-
-        with stdlib_io.open(f1, 'r') as f:
-            nt.assert_equal(f.read(), u'Overwritten')
-
-        if os.name != 'nt':
-            mode = stat.S_IMODE(os.stat(f1).st_mode)
-            nt.assert_equal(mode, orig_mode)
-
-        if have_symlink:
-            # Check that writing over a file preserves a symlink
-            with atomic_writing(f2) as f:
-                f.write(u'written from symlink')
-            
-            with stdlib_io.open(f1, 'r') as f:
-                nt.assert_equal(f.read(), u'written from symlink')
-
-def _save_umask():
-    global umask
-    umask = os.umask(0)
-    os.umask(umask)
-
-def _restore_umask():
-    os.umask(umask)
-
-@skip_win32
[e-mail]_setup(_save_umask, _restore_umask)
-def test_atomic_writing_umask():
-    with TemporaryDirectory() as td:
-        os.umask(0o022)
-        f1 = os.path.join(td, '1')
-        with atomic_writing(f1) as f:
-            f.write(u'1')
-        mode = stat.S_IMODE(os.stat(f1).st_mode)
-        nt.assert_equal(mode, 0o644, '{:o} != 644'.format(mode))
-
-        os.umask(0o057)
-        f2 = os.path.join(td, '2')
-        with atomic_writing(f2) as f:
-            f.write(u'2')
-        mode = stat.S_IMODE(os.stat(f2).st_mode)
-        nt.assert_equal(mode, 0o620, '{:o} != 620'.format(mode))
-
-
-def test_atomic_writing_newlines():
-    with TemporaryDirectory() as td:
-        path = os.path.join(td, 'testfile')
-        
-        lf = u'a\nb\nc\n'
-        plat = lf.replace(u'\n', os.linesep)
-        crlf = lf.replace(u'\n', u'\r\n')
-        
-        # test default
-        with stdlib_io.open(path, 'w') as f:
-            f.write(lf)
-        with stdlib_io.open(path, 'r', newline='') as f:
-            read = f.read()
-        nt.assert_equal(read, plat)
-        
-        # test newline=LF
-        with stdlib_io.open(path, 'w', newline='\n') as f:
-            f.write(lf)
-        with stdlib_io.open(path, 'r', newline='') as f:
-            read = f.read()
-        nt.assert_equal(read, lf)
-        
-        # test newline=CRLF
-        with atomic_writing(path, newline='\r\n') as f:
-            f.write(lf)
-        with stdlib_io.open(path, 'r', newline='') as f:
-            read = f.read()
-        nt.assert_equal(read, crlf)
-        
-        # test newline=no convert
-        text = u'crlf\r\ncr\rlf\n'
-        with atomic_writing(path, newline='') as f:
-            f.write(text)
-        with stdlib_io.open(path, 'r', newline='') as f:
-            read = f.read()
-        nt.assert_equal(read, text)
+

jupyter_client/session.py

@@ -1,889 +1,891 b''
 """Session object for building, serializing, sending, and receiving messages in
 IPython. The Session object supports serialization, HMAC signatures, and
 metadata on messages.
 
 Also defined here are utilities for working with Sessions:
 * A SessionFactory to be used as a base class for configurables that work with
 Sessions.
 * A Message object for convenience that allows attribute-access to the msg dict.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import hashlib
 import hmac
 import logging
 import os
 import pprint
 import random
 import uuid
 import warnings
 from datetime import datetime
 
 try:
     import cPickle
     pickle = cPickle
 except:
     cPickle = None
     import pickle
 
 try:
     # py3
     PICKLE_PROTOCOL = pickle.DEFAULT_PROTOCOL
 except AttributeError:
     PICKLE_PROTOCOL = pickle.HIGHEST_PROTOCOL
 
 try:
     # We are using compare_digest to limit the surface of timing attacks
     from hmac import compare_digest
 except ImportError:
     # Python < 2.7.7: When digests don't match no feedback is provided,
     # limiting the surface of attack
     def compare_digest(a,b): return a == b
 
 import zmq
 from zmq.utils import jsonapi
 from zmq.eventloop.ioloop import IOLoop
 from zmq.eventloop.zmqstream import ZMQStream
 
 from IPython.core.release import kernel_protocol_version
 from IPython.config.configurable import Configurable, LoggingConfigurable
-from IPython.utils import io
 from IPython.utils.importstring import import_item
 from jupyter_client.jsonutil import extract_dates, squash_dates, date_default
 from IPython.utils.py3compat import (str_to_bytes, str_to_unicode, unicode_type,
                                      iteritems)
 from IPython.utils.traitlets import (CBytes, Unicode, Bool, Any, Instance, Set,
                                         DottedObjectName, CUnicode, Dict, Integer,
                                         TraitError,
 )
 from jupyter_client.adapter import adapt
+from traitlets.log import get_logger
+
 
 #-----------------------------------------------------------------------------
 # utility functions
 #-----------------------------------------------------------------------------
 
 def squash_unicode(obj):
     """coerce unicode back to bytestrings."""
     if isinstance(obj,dict):
         for key in obj.keys():
             obj[key] = squash_unicode(obj[key])
             if isinstance(key, unicode_type):
                 obj[squash_unicode(key)] = obj.pop(key)
     elif isinstance(obj, list):
         for i,v in enumerate(obj):
             obj[i] = squash_unicode(v)
     elif isinstance(obj, unicode_type):
         obj = obj.encode('utf8')
     return obj
 
 #-----------------------------------------------------------------------------
 # globals and defaults
 #-----------------------------------------------------------------------------
 
 # default values for the thresholds:
 MAX_ITEMS = 64
 MAX_BYTES = 1024
 
 # ISO8601-ify datetime objects
 # allow unicode
 # disallow nan, because it's not actually valid JSON
 json_packer = lambda obj: jsonapi.dumps(obj, default=date_default,
     ensure_ascii=False, allow_nan=False,
 )
 json_unpacker = lambda s: jsonapi.loads(s)
 
 pickle_packer = lambda o: pickle.dumps(squash_dates(o), PICKLE_PROTOCOL)
 pickle_unpacker = pickle.loads
 
 default_packer = json_packer
 default_unpacker = json_unpacker
 
 DELIM = b"<IDS|MSG>"
 # singleton dummy tracker, which will always report as done
 DONE = zmq.MessageTracker()
 
 #-----------------------------------------------------------------------------
 # Mixin tools for apps that use Sessions
 #-----------------------------------------------------------------------------
 
 session_aliases = dict(
     ident = 'Session.session',
     user = 'Session.username',
     keyfile = 'Session.keyfile',
 )
 
 session_flags  = {
     'secure' : ({'Session' : { 'key' : str_to_bytes(str(uuid.uuid4())),
                             'keyfile' : '' }},
         """Use HMAC digests for authentication of messages.
         Setting this flag will generate a new UUID to use as the HMAC key.
         """),
     'no-secure' : ({'Session' : { 'key' : b'', 'keyfile' : '' }},
         """Don't authenticate messages."""),
 }
 
 def default_secure(cfg):
     """Set the default behavior for a config environment to be secure.
 
     If Session.key/keyfile have not been set, set Session.key to
     a new random UUID.
     """
     warnings.warn("default_secure is deprecated", DeprecationWarning)
     if 'Session' in cfg:
         if 'key' in cfg.Session or 'keyfile' in cfg.Session:
             return
     # key/keyfile not specified, generate new UUID:
     cfg.Session.key = str_to_bytes(str(uuid.uuid4()))
 
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 class SessionFactory(LoggingConfigurable):
     """The Base class for configurables that have a Session, Context, logger,
     and IOLoop.
     """
 
     logname = Unicode('')
     def _logname_changed(self, name, old, new):
         self.log = logging.getLogger(new)
 
     # not configurable:
     context = Instance('zmq.Context')
     def _context_default(self):
         return zmq.Context.instance()
 
     session = Instance('jupyter_client.session.Session',
                        allow_none=True)
 
     loop = Instance('zmq.eventloop.ioloop.IOLoop')
     def _loop_default(self):
         return IOLoop.instance()
 
     def __init__(self, **kwargs):
         super(SessionFactory, self).__init__(**kwargs)
 
         if self.session is None:
             # construct the session
             self.session = Session(**kwargs)
 
 
 class Message(object):
     """A simple message object that maps dict keys to attributes.
 
     A Message can be created from a dict and a dict from a Message instance
     simply by calling dict(msg_obj)."""
 
     def __init__(self, msg_dict):
         dct = self.__dict__
         for k, v in iteritems(dict(msg_dict)):
             if isinstance(v, dict):
                 v = Message(v)
             dct[k] = v
 
     # Having this iterator lets dict(msg_obj) work out of the box.
     def __iter__(self):
         return iter(iteritems(self.__dict__))
 
     def __repr__(self):
         return repr(self.__dict__)
 
     def __str__(self):
         return pprint.pformat(self.__dict__)
 
     def __contains__(self, k):
         return k in self.__dict__
 
     def __getitem__(self, k):
         return self.__dict__[k]
 
 
 def msg_header(msg_id, msg_type, username, session):
     date = datetime.now()
     version = kernel_protocol_version
     return locals()
 
 def extract_header(msg_or_header):
     """Given a message or header, return the header."""
     if not msg_or_header:
         return {}
     try:
         # See if msg_or_header is the entire message.
         h = msg_or_header['header']
     except KeyError:
         try:
             # See if msg_or_header is just the header
             h = msg_or_header['msg_id']
         except KeyError:
             raise
         else:
             h = msg_or_header
     if not isinstance(h, dict):
         h = dict(h)
     return h
 
 class Session(Configurable):
     """Object for handling serialization and sending of messages.
 
     The Session object handles building messages and sending them
     with ZMQ sockets or ZMQStream objects.  Objects can communicate with each
     other over the network via Session objects, and only need to work with the
     dict-based IPython message spec. The Session will handle
     serialization/deserialization, security, and metadata.
 
     Sessions support configurable serialization via packer/unpacker traits,
     and signing with HMAC digests via the key/keyfile traits.
 
     Parameters
     ----------
 
     debug : bool
         whether to trigger extra debugging statements
     packer/unpacker : str : 'json', 'pickle' or import_string
         importstrings for methods to serialize message parts.  If just
         'json' or 'pickle', predefined JSON and pickle packers will be used.
         Otherwise, the entire importstring must be used.
 
         The functions must accept at least valid JSON input, and output *bytes*.
 
         For example, to use msgpack:
         packer = 'msgpack.packb', unpacker='msgpack.unpackb'
     pack/unpack : callables
         You can also set the pack/unpack callables for serialization directly.
     session : bytes
         the ID of this Session object.  The default is to generate a new UUID.
     username : unicode
         username added to message headers.  The default is to ask the OS.
     key : bytes
         The key used to initialize an HMAC signature.  If unset, messages
         will not be signed or checked.
     keyfile : filepath
         The file containing a key.  If this is set, `key` will be initialized
         to the contents of the file.
 
     """
 
     debug=Bool(False, config=True, help="""Debug output in the Session""")
 
     packer = DottedObjectName('json',config=True,
             help="""The name of the packer for serializing messages.
             Should be one of 'json', 'pickle', or an import name
             for a custom callable serializer.""")
     def _packer_changed(self, name, old, new):
         if new.lower() == 'json':
             self.pack = json_packer
             self.unpack = json_unpacker
             self.unpacker = new
         elif new.lower() == 'pickle':
             self.pack = pickle_packer
             self.unpack = pickle_unpacker
             self.unpacker = new
         else:
             self.pack = import_item(str(new))
 
     unpacker = DottedObjectName('json', config=True,
         help="""The name of the unpacker for unserializing messages.
         Only used with custom functions for `packer`.""")
     def _unpacker_changed(self, name, old, new):
         if new.lower() == 'json':
             self.pack = json_packer
             self.unpack = json_unpacker
             self.packer = new
         elif new.lower() == 'pickle':
             self.pack = pickle_packer
             self.unpack = pickle_unpacker
             self.packer = new
         else:
             self.unpack = import_item(str(new))
 
     session = CUnicode(u'', config=True,
         help="""The UUID identifying this session.""")
     def _session_default(self):
         u = unicode_type(uuid.uuid4())
         self.bsession = u.encode('ascii')
         return u
 
     def _session_changed(self, name, old, new):
         self.bsession = self.session.encode('ascii')
 
     # bsession is the session as bytes
     bsession = CBytes(b'')
 
     username = Unicode(str_to_unicode(os.environ.get('USER', 'username')),
         help="""Username for the Session. Default is your system username.""",
         config=True)
 
     metadata = Dict({}, config=True,
         help="""Metadata dictionary, which serves as the default top-level metadata dict for each message.""")
 
     # if 0, no adapting to do.
     adapt_version = Integer(0)
 
     # message signature related traits:
 
     key = CBytes(config=True,
         help="""execution key, for signing messages.""")
     def _key_default(self):
         return str_to_bytes(str(uuid.uuid4()))
 
     def _key_changed(self):
         self._new_auth()
 
     signature_scheme = Unicode('hmac-sha256', config=True,
         help="""The digest scheme used to construct the message signatures.
         Must have the form 'hmac-HASH'.""")
     def _signature_scheme_changed(self, name, old, new):
         if not new.startswith('hmac-'):
             raise TraitError("signature_scheme must start with 'hmac-', got %r" % new)
         hash_name = new.split('-', 1)[1]
         try:
             self.digest_mod = getattr(hashlib, hash_name)
         except AttributeError:
             raise TraitError("hashlib has no such attribute: %s" % hash_name)
         self._new_auth()
 
     digest_mod = Any()
     def _digest_mod_default(self):
         return hashlib.sha256
     
     auth = Instance(hmac.HMAC, allow_none=True)
     
     def _new_auth(self):
         if self.key:
             self.auth = hmac.HMAC(self.key, digestmod=self.digest_mod)
         else:
             self.auth = None
 
     digest_history = Set()
     digest_history_size = Integer(2**16, config=True,
         help="""The maximum number of digests to remember.
 
         The digest history will be culled when it exceeds this value.
         """
     )
 
     keyfile = Unicode('', config=True,
         help="""path to file containing execution key.""")
     def _keyfile_changed(self, name, old, new):
         with open(new, 'rb') as f:
             self.key = f.read().strip()
 
     # for protecting against sends from forks
     pid = Integer()
 
     # serialization traits:
 
     pack = Any(default_packer) # the actual packer function
     def _pack_changed(self, name, old, new):
         if not callable(new):
             raise TypeError("packer must be callable, not %s"%type(new))
 
     unpack = Any(default_unpacker) # the actual packer function
     def _unpack_changed(self, name, old, new):
         # unpacker is not checked - it is assumed to be
         if not callable(new):
             raise TypeError("unpacker must be callable, not %s"%type(new))
 
     # thresholds:
     copy_threshold = Integer(2**16, config=True,
         help="Threshold (in bytes) beyond which a buffer should be sent without copying.")
     buffer_threshold = Integer(MAX_BYTES, config=True,
         help="Threshold (in bytes) beyond which an object's buffer should be extracted to avoid pickling.")
     item_threshold = Integer(MAX_ITEMS, config=True,
         help="""The maximum number of items for a container to be introspected for custom serialization.
         Containers larger than this are pickled outright.
         """
     )
 
 
     def __init__(self, **kwargs):
         """create a Session object
 
         Parameters
         ----------
 
         debug : bool
             whether to trigger extra debugging statements
         packer/unpacker : str : 'json', 'pickle' or import_string
             importstrings for methods to serialize message parts.  If just
             'json' or 'pickle', predefined JSON and pickle packers will be used.
             Otherwise, the entire importstring must be used.
 
             The functions must accept at least valid JSON input, and output
             *bytes*.
 
             For example, to use msgpack:
             packer = 'msgpack.packb', unpacker='msgpack.unpackb'
         pack/unpack : callables
             You can also set the pack/unpack callables for serialization
             directly.
         session : unicode (must be ascii)
             the ID of this Session object.  The default is to generate a new
             UUID.
         bsession : bytes
             The session as bytes
         username : unicode
             username added to message headers.  The default is to ask the OS.
         key : bytes
             The key used to initialize an HMAC signature.  If unset, messages
             will not be signed or checked.
         signature_scheme : str
             The message digest scheme. Currently must be of the form 'hmac-HASH',
             where 'HASH' is a hashing function available in Python's hashlib.
             The default is 'hmac-sha256'.
             This is ignored if 'key' is empty.
         keyfile : filepath
             The file containing a key.  If this is set, `key` will be
             initialized to the contents of the file.
         """
         super(Session, self).__init__(**kwargs)
         self._check_packers()
         self.none = self.pack({})
         # ensure self._session_default() if necessary, so bsession is defined:
         self.session
         self.pid = os.getpid()
         self._new_auth()
 
     @property
     def msg_id(self):
         """always return new uuid"""
         return str(uuid.uuid4())
 
     def _check_packers(self):
         """check packers for datetime support."""
         pack = self.pack
         unpack = self.unpack
 
         # check simple serialization
         msg = dict(a=[1,'hi'])
         try:
             packed = pack(msg)
         except Exception as e:
             msg = "packer '{packer}' could not serialize a simple message: {e}{jsonmsg}"
             if self.packer == 'json':
                 jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod
             else:
                 jsonmsg = ""
             raise ValueError(
                 msg.format(packer=self.packer, e=e, jsonmsg=jsonmsg)
             )
 
         # ensure packed message is bytes
         if not isinstance(packed, bytes):
             raise ValueError("message packed to %r, but bytes are required"%type(packed))
 
         # check that unpack is pack's inverse
         try:
             unpacked = unpack(packed)
             assert unpacked == msg
         except Exception as e:
             msg = "unpacker '{unpacker}' could not handle output from packer '{packer}': {e}{jsonmsg}"
             if self.packer == 'json':
                 jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod
             else:
                 jsonmsg = ""
             raise ValueError(
                 msg.format(packer=self.packer, unpacker=self.unpacker, e=e, jsonmsg=jsonmsg)
             )
 
         # check datetime support
         msg = dict(t=datetime.now())
         try:
             unpacked = unpack(pack(msg))
             if isinstance(unpacked['t'], datetime):
                 raise ValueError("Shouldn't deserialize to datetime")
         except Exception:
             self.pack = lambda o: pack(squash_dates(o))
             self.unpack = lambda s: unpack(s)
 
     def msg_header(self, msg_type):
         return msg_header(self.msg_id, msg_type, self.username, self.session)
 
     def msg(self, msg_type, content=None, parent=None, header=None, metadata=None):
         """Return the nested message dict.
 
         This format is different from what is sent over the wire. The
         serialize/deserialize methods converts this nested message dict to the wire
         format, which is a list of message parts.
         """
         msg = {}
         header = self.msg_header(msg_type) if header is None else header
         msg['header'] = header
         msg['msg_id'] = header['msg_id']
         msg['msg_type'] = header['msg_type']
         msg['parent_header'] = {} if parent is None else extract_header(parent)
         msg['content'] = {} if content is None else content
         msg['metadata'] = self.metadata.copy()
         if metadata is not None:
             msg['metadata'].update(metadata)
         return msg
 
     def sign(self, msg_list):
         """Sign a message with HMAC digest. If no auth, return b''.
 
         Parameters
         ----------
         msg_list : list
             The [p_header,p_parent,p_content] part of the message list.
         """
         if self.auth is None:
             return b''
         h = self.auth.copy()
         for m in msg_list:
             h.update(m)
         return str_to_bytes(h.hexdigest())
 
     def serialize(self, msg, ident=None):
         """Serialize the message components to bytes.
 
         This is roughly the inverse of deserialize. The serialize/deserialize
         methods work with full message lists, whereas pack/unpack work with
         the individual message parts in the message list.
 
         Parameters
         ----------
         msg : dict or Message
             The next message dict as returned by the self.msg method.
 
         Returns
         -------
         msg_list : list
             The list of bytes objects to be sent with the format::
 
                 [ident1, ident2, ..., DELIM, HMAC, p_header, p_parent,
                  p_metadata, p_content, buffer1, buffer2, ...]
 
             In this list, the ``p_*`` entities are the packed or serialized
             versions, so if JSON is used, these are utf8 encoded JSON strings.
         """
         content = msg.get('content', {})
         if content is None:
             content = self.none
         elif isinstance(content, dict):
             content = self.pack(content)
         elif isinstance(content, bytes):
             # content is already packed, as in a relayed message
             pass
         elif isinstance(content, unicode_type):
             # should be bytes, but JSON often spits out unicode
             content = content.encode('utf8')
         else:
             raise TypeError("Content incorrect type: %s"%type(content))
 
         real_message = [self.pack(msg['header']),
                         self.pack(msg['parent_header']),
                         self.pack(msg['metadata']),
                         content,
         ]
 
         to_send = []
 
         if isinstance(ident, list):
             # accept list of idents
             to_send.extend(ident)
         elif ident is not None:
             to_send.append(ident)
         to_send.append(DELIM)
 
         signature = self.sign(real_message)
         to_send.append(signature)
 
         to_send.extend(real_message)
 
         return to_send
 
     def send(self, stream, msg_or_type, content=None, parent=None, ident=None,
              buffers=None, track=False, header=None, metadata=None):
         """Build and send a message via stream or socket.
 
         The message format used by this function internally is as follows:
 
         [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content,
          buffer1,buffer2,...]
 
         The serialize/deserialize methods convert the nested message dict into this
         format.
 
         Parameters
         ----------
 
         stream : zmq.Socket or ZMQStream
             The socket-like object used to send the data.
         msg_or_type : str or Message/dict
             Normally, msg_or_type will be a msg_type unless a message is being
             sent more than once. If a header is supplied, this can be set to
             None and the msg_type will be pulled from the header.
 
         content : dict or None
             The content of the message (ignored if msg_or_type is a message).
         header : dict or None
             The header dict for the message (ignored if msg_to_type is a message).
         parent : Message or dict or None
             The parent or parent header describing the parent of this message
             (ignored if msg_or_type is a message).
         ident : bytes or list of bytes
             The zmq.IDENTITY routing path.
         metadata : dict or None
             The metadata describing the message
         buffers : list or None
             The already-serialized buffers to be appended to the message.
         track : bool
             Whether to track.  Only for use with Sockets, because ZMQStream
             objects cannot track messages.
 
 
         Returns
         -------
         msg : dict
             The constructed message.
         """
         if not isinstance(stream, zmq.Socket):
             # ZMQStreams and dummy sockets do not support tracking.
             track = False
 
         if isinstance(msg_or_type, (Message, dict)):
             # We got a Message or message dict, not a msg_type so don't
             # build a new Message.
             msg = msg_or_type
             buffers = buffers or msg.get('buffers', [])
         else:
             msg = self.msg(msg_or_type, content=content, parent=parent,
                            header=header, metadata=metadata)
         if not os.getpid() == self.pid:
-            io.rprint("WARNING: attempted to send message from fork")
-            io.rprint(msg)
+            get_logger().warn("WARNING: attempted to send message from fork\n%s",
+                msg
+            )
             return
         buffers = [] if buffers is None else buffers
         if self.adapt_version:
             msg = adapt(msg, self.adapt_version)
         to_send = self.serialize(msg, ident)
         to_send.extend(buffers)
         longest = max([ len(s) for s in to_send ])
         copy = (longest < self.copy_threshold)
 
         if buffers and track and not copy:
             # only really track when we are doing zero-copy buffers
             tracker = stream.send_multipart(to_send, copy=False, track=True)
         else:
             # use dummy tracker, which will be done immediately
             tracker = DONE
             stream.send_multipart(to_send, copy=copy)
 
         if self.debug:
             pprint.pprint(msg)
             pprint.pprint(to_send)
             pprint.pprint(buffers)
 
         msg['tracker'] = tracker
 
         return msg
 
     def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None):
         """Send a raw message via ident path.
 
         This method is used to send a already serialized message.
 
         Parameters
         ----------
         stream : ZMQStream or Socket
             The ZMQ stream or socket to use for sending the message.
         msg_list : list
             The serialized list of messages to send. This only includes the
             [p_header,p_parent,p_metadata,p_content,buffer1,buffer2,...] portion of
             the message.
         ident : ident or list
             A single ident or a list of idents to use in sending.
         """
         to_send = []
         if isinstance(ident, bytes):
             ident = [ident]
         if ident is not None:
             to_send.extend(ident)
 
         to_send.append(DELIM)
         to_send.append(self.sign(msg_list))
         to_send.extend(msg_list)
         stream.send_multipart(to_send, flags, copy=copy)
 
     def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):
         """Receive and unpack a message.
 
         Parameters
         ----------
         socket : ZMQStream or Socket
             The socket or stream to use in receiving.
 
         Returns
         -------
         [idents], msg
             [idents] is a list of idents and msg is a nested message dict of
             same format as self.msg returns.
         """
         if isinstance(socket, ZMQStream):
             socket = socket.socket
         try:
             msg_list = socket.recv_multipart(mode, copy=copy)
         except zmq.ZMQError as e:
             if e.errno == zmq.EAGAIN:
                 # We can convert EAGAIN to None as we know in this case
                 # recv_multipart won't return None.
                 return None,None
             else:
                 raise
         # split multipart message into identity list and message dict
         # invalid large messages can cause very expensive string comparisons
         idents, msg_list = self.feed_identities(msg_list, copy)
         try:
             return idents, self.deserialize(msg_list, content=content, copy=copy)
         except Exception as e:
             # TODO: handle it
             raise e
 
     def feed_identities(self, msg_list, copy=True):
         """Split the identities from the rest of the message.
 
         Feed until DELIM is reached, then return the prefix as idents and
         remainder as msg_list. This is easily broken by setting an IDENT to DELIM,
         but that would be silly.
 
         Parameters
         ----------
         msg_list : a list of Message or bytes objects
             The message to be split.
         copy : bool
             flag determining whether the arguments are bytes or Messages
 
         Returns
         -------
         (idents, msg_list) : two lists
             idents will always be a list of bytes, each of which is a ZMQ
             identity. msg_list will be a list of bytes or zmq.Messages of the
             form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and
             should be unpackable/unserializable via self.deserialize at this
             point.
         """
         if copy:
             idx = msg_list.index(DELIM)
             return msg_list[:idx], msg_list[idx+1:]
         else:
             failed = True
             for idx,m in enumerate(msg_list):
                 if m.bytes == DELIM:
                     failed = False
                     break
             if failed:
                 raise ValueError("DELIM not in msg_list")
             idents, msg_list = msg_list[:idx], msg_list[idx+1:]
             return [m.bytes for m in idents], msg_list
 
     def _add_digest(self, signature):
         """add a digest to history to protect against replay attacks"""
         if self.digest_history_size == 0:
             # no history, never add digests
             return
 
         self.digest_history.add(signature)
         if len(self.digest_history) > self.digest_history_size:
             # threshold reached, cull 10%
             self._cull_digest_history()
 
     def _cull_digest_history(self):
         """cull the digest history
 
         Removes a randomly selected 10% of the digest history
         """
         current = len(self.digest_history)
         n_to_cull = max(int(current // 10), current - self.digest_history_size)
         if n_to_cull >= current:
             self.digest_history = set()
             return
         to_cull = random.sample(self.digest_history, n_to_cull)
         self.digest_history.difference_update(to_cull)
 
     def deserialize(self, msg_list, content=True, copy=True):
         """Unserialize a msg_list to a nested message dict.
 
         This is roughly the inverse of serialize. The serialize/deserialize
         methods work with full message lists, whereas pack/unpack work with
         the individual message parts in the message list.
 
         Parameters
         ----------
         msg_list : list of bytes or Message objects
             The list of message parts of the form [HMAC,p_header,p_parent,
             p_metadata,p_content,buffer1,buffer2,...].
         content : bool (True)
             Whether to unpack the content dict (True), or leave it packed
             (False).
         copy : bool (True)
             Whether msg_list contains bytes (True) or the non-copying Message
             objects in each place (False).
 
         Returns
         -------
         msg : dict
             The nested message dict with top-level keys [header, parent_header,
             content, buffers].  The buffers are returned as memoryviews.
         """
         minlen = 5
         message = {}
         if not copy:
             # pyzmq didn't copy the first parts of the message, so we'll do it
             for i in range(minlen):
                 msg_list[i] = msg_list[i].bytes
         if self.auth is not None:
             signature = msg_list[0]
             if not signature:
                 raise ValueError("Unsigned Message")
             if signature in self.digest_history:
                 raise ValueError("Duplicate Signature: %r" % signature)
             self._add_digest(signature)
             check = self.sign(msg_list[1:5])
             if not compare_digest(signature, check):
                 raise ValueError("Invalid Signature: %r" % signature)
         if not len(msg_list) >= minlen:
             raise TypeError("malformed message, must have at least %i elements"%minlen)
         header = self.unpack(msg_list[1])
         message['header'] = extract_dates(header)
         message['msg_id'] = header['msg_id']
         message['msg_type'] = header['msg_type']
         message['parent_header'] = extract_dates(self.unpack(msg_list[2]))
         message['metadata'] = self.unpack(msg_list[3])
         if content:
             message['content'] = self.unpack(msg_list[4])
         else:
             message['content'] = msg_list[4]
         buffers = [memoryview(b) for b in msg_list[5:]]
         if buffers and buffers[0].shape is None:
             # force copy to workaround pyzmq #646
             buffers = [memoryview(b.bytes) for b in msg_list[5:]]
         message['buffers'] = buffers
         # adapt to the current version
         return adapt(message)
 
     def unserialize(self, *args, **kwargs):
         warnings.warn(
             "Session.unserialize is deprecated. Use Session.deserialize.",
             DeprecationWarning,
         )
         return self.deserialize(*args, **kwargs)
 
 
 def test_msg2obj():
     am = dict(x=1)
     ao = Message(am)
     assert ao.x == am['x']
 
     am['y'] = dict(z=1)
     ao = Message(am)
     assert ao.y.z == am['y']['z']
 
     k1, k2 = 'y', 'z'
     assert ao[k1][k2] == am[k1][k2]
 
     am2 = dict(ao)
     assert am['x'] == am2['x']
     assert am['y']['z'] == am2['y']['z']

jupyter_nbconvert/utils/pandoc.py

@@ -1,141 +1,128 b''
 """Utility for calling pandoc"""
-#-----------------------------------------------------------------------------
-# Copyright (c) 2014 the IPython Development Team.
-#
+# Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-from __future__ import print_function
+from __future__ import print_function, absolute_import
 
-# Stdlib imports
 import subprocess
 import warnings
 import re
 from io import TextIOWrapper, BytesIO
 
-# IPython imports
 from IPython.utils.py3compat import cast_bytes
 from IPython.utils.version import check_version
 from IPython.utils.process import is_cmd_found, FindCmdError
 
 from .exceptions import ConversionException
 
-#-----------------------------------------------------------------------------
-# Classes and functions
-#-----------------------------------------------------------------------------
 _minimal_version = "1.12.1"
 
 def pandoc(source, fmt, to, extra_args=None, encoding='utf-8'):
     """Convert an input string in format `from` to format `to` via pandoc.
 
     Parameters
     ----------
     source : string
       Input string, assumed to be valid format `from`.
     fmt : string
       The name of the input format (markdown, etc.)
     to : string
       The name of the output format (html, etc.)
 
     Returns
     -------
     out : unicode
       Output as returned by pandoc.
 
     Raises
     ------
     PandocMissing
       If pandoc is not installed.
     
     Any error messages generated by pandoc are printed to stderr.
 
     """
     cmd = ['pandoc', '-f', fmt, '-t', to]
     if extra_args:
         cmd.extend(extra_args)
 
     # this will raise an exception that will pop us out of here
     check_pandoc_version()
     
     # we can safely continue
     p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
     out, _ = p.communicate(cast_bytes(source, encoding))
     out = TextIOWrapper(BytesIO(out), encoding, 'replace').read()
     return out.rstrip('\n')
 
 
 def get_pandoc_version():
     """Gets the Pandoc version if Pandoc is installed.
     
     If the minimal version is not met, it will probe Pandoc for its version, cache it and return that value.
     If the minimal version is met, it will return the cached version and stop probing Pandoc 
     (unless :func:`clean_cache()` is called).
 
     Raises
     ------
     PandocMissing
       If pandoc is unavailable.
     """
     global __version
 
     if __version is None:
         if not is_cmd_found('pandoc'):
             raise PandocMissing()
 
         out = subprocess.check_output(['pandoc', '-v'],
                                       universal_newlines=True)
         out_lines = out.splitlines()
         version_pattern = re.compile(r"^\d+(\.\d+){1,}$")
         for tok in out_lines[0].split():
             if version_pattern.match(tok):
                 __version = tok
                 break
     return __version
 
 
 def check_pandoc_version():
     """Returns True if minimal pandoc version is met.
 
     Raises
     ------
     PandocMissing
       If pandoc is unavailable.
     """
     v = get_pandoc_version()
     if v is None:
         warnings.warn("Sorry, we cannot determine the version of pandoc.\n"
                       "Please consider reporting this issue and include the"
                       "output of pandoc --version.\nContinuing...",
                       RuntimeWarning, stacklevel=2)
         return False
     ok = check_version(v , _minimal_version )
     if not ok:
         warnings.warn( "You are using an old version of pandoc (%s)\n" % v + 
                        "Recommended version is %s.\nTry updating." % _minimal_version + 
                        "http://johnmacfarlane.net/pandoc/installing.html.\nContinuing with doubts...",
                        RuntimeWarning, stacklevel=2)
     return ok
 
 #-----------------------------------------------------------------------------
 # Exception handling
 #-----------------------------------------------------------------------------
 class PandocMissing(ConversionException):
     """Exception raised when Pandoc is missing. """
     def __init__(self, *args, **kwargs):
         super(PandocMissing, self).__init__( "Pandoc wasn't found.\n" +
                                              "Please check that pandoc is installed:\n" +
                                              "http://johnmacfarlane.net/pandoc/installing.html" )
 
 #-----------------------------------------------------------------------------
 # Internal state management
 #-----------------------------------------------------------------------------
 def clean_cache():
     global __version
     __version = None
 
 __version = None

jupyter_nbconvert/writers/stdout.py

@@ -1,34 +1,23 b''
 """
 Contains Stdout writer
 """
-#-----------------------------------------------------------------------------
-#Copyright (c) 2013, the IPython Development Team.
-#
-#Distributed under the terms of the Modified BSD License.
-#
-#The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
 
-from IPython.utils import io
+from jupyter_nbconvert.utils import io
 from .base import WriterBase
 
-#-----------------------------------------------------------------------------
-# Classes
-#-----------------------------------------------------------------------------
 
 class StdoutWriter(WriterBase):
     """Consumes output from nbconvert export...() methods and writes to the 
     stdout stream."""
 
 
     def write(self, output, resources, **kw):
         """
         Consume and write Jinja output.
 
         See base for more...
         """
         io.unicode_std_stream().write(output)