From a229520abeece08d7e098ee788850c9bd6f34639 2015-04-03 18:12:58 From: Thomas Kluyver Date: 2015-04-03 18:12:58 Subject: [PATCH] Merge pull request #8240 from minrk/split-io split utils.io --- diff --git a/IPython/html/services/contents/fileio.py b/IPython/html/services/contents/fileio.py index 0f7ff5c..7fa09c2 100644 --- a/IPython/html/services/contents/fileio.py +++ b/IPython/html/services/contents/fileio.py @@ -11,6 +11,7 @@ import errno import io import os import shutil +import tempfile from tornado.web import HTTPError @@ -19,10 +20,91 @@ from IPython.html.utils import ( to_os_path, ) from IPython import nbformat -from IPython.utils.io import atomic_writing from IPython.utils.py3compat import str_to_unicode +def _copy_metadata(src, dst): + """Copy the set of metadata we want for atomic_writing. + + Permission bits and flags. We'd like to copy file ownership as well, but we + can't do that. + """ + shutil.copymode(src, dst) + st = os.stat(src) + if hasattr(os, 'chflags') and hasattr(st, 'st_flags'): + os.chflags(dst, st.st_flags) + +@contextmanager +def atomic_writing(path, text=True, encoding='utf-8', **kwargs): + """Context manager to write to a file only if the entire write is successful. + + This works by creating a temporary file in the same directory, and renaming + it over the old file if the context is exited without an error. If other + file names are hard linked to the target file, this relationship will not be + preserved. + + On Windows, there is a small chink in the atomicity: the target file is + deleted before renaming the temporary file over it. This appears to be + unavoidable. + + Parameters + ---------- + path : str + The target file to write to. + + text : bool, optional + Whether to open the file in text mode (i.e. to write unicode). Default is + True. + + encoding : str, optional + The encoding to use for files opened in text mode. Default is UTF-8. + + **kwargs + Passed to :func:`io.open`. + """ + # realpath doesn't work on Windows: http://bugs.python.org/issue9949 + # Luckily, we only need to resolve the file itself being a symlink, not + # any of its directories, so this will suffice: + if os.path.islink(path): + path = os.path.join(os.path.dirname(path), os.readlink(path)) + + dirname, basename = os.path.split(path) + tmp_dir = tempfile.mkdtemp(prefix=basename, dir=dirname) + tmp_path = os.path.join(tmp_dir, basename) + if text: + fileobj = io.open(tmp_path, 'w', encoding=encoding, **kwargs) + else: + fileobj = io.open(tmp_path, 'wb', **kwargs) + + try: + yield fileobj + except: + fileobj.close() + shutil.rmtree(tmp_dir) + raise + + # Flush to disk + fileobj.flush() + os.fsync(fileobj.fileno()) + + # Written successfully, now rename it + fileobj.close() + + # Copy permission bits, access time, etc. + try: + _copy_metadata(path, tmp_path) + except OSError: + # e.g. the file didn't already exist. Ignore any failure to copy metadata + pass + + if os.name == 'nt' and os.path.exists(path): + # Rename over existing file doesn't work on Windows + os.remove(path) + + os.rename(tmp_path, path) + shutil.rmtree(tmp_dir) + + class FileManagerMixin(object): """ Mixin for ContentsAPI classes that interact with the filesystem. diff --git a/IPython/html/services/contents/tests/test_fileio.py b/IPython/html/services/contents/tests/test_fileio.py new file mode 100644 index 0000000..b517280 --- /dev/null +++ b/IPython/html/services/contents/tests/test_fileio.py @@ -0,0 +1,131 @@ +# encoding: utf-8 +"""Tests for file IO""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import io as stdlib_io +import os.path +import stat + +import nose.tools as nt + +from IPython.testing.decorators import skip_win32 +from ..fileio import atomic_writing + +from IPython.utils.tempdir import TemporaryDirectory + +umask = 0 + +def test_atomic_writing(): + class CustomExc(Exception): pass + + with TemporaryDirectory() as td: + f1 = os.path.join(td, 'penguin') + with stdlib_io.open(f1, 'w') as f: + f.write(u'Before') + + if os.name != 'nt': + os.chmod(f1, 0o701) + orig_mode = stat.S_IMODE(os.stat(f1).st_mode) + + f2 = os.path.join(td, 'flamingo') + try: + os.symlink(f1, f2) + have_symlink = True + except (AttributeError, NotImplementedError, OSError): + # AttributeError: Python doesn't support it + # NotImplementedError: The system doesn't support it + # OSError: The user lacks the privilege (Windows) + have_symlink = False + + with nt.assert_raises(CustomExc): + with atomic_writing(f1) as f: + f.write(u'Failing write') + raise CustomExc + + # Because of the exception, the file should not have been modified + with stdlib_io.open(f1, 'r') as f: + nt.assert_equal(f.read(), u'Before') + + with atomic_writing(f1) as f: + f.write(u'Overwritten') + + with stdlib_io.open(f1, 'r') as f: + nt.assert_equal(f.read(), u'Overwritten') + + if os.name != 'nt': + mode = stat.S_IMODE(os.stat(f1).st_mode) + nt.assert_equal(mode, orig_mode) + + if have_symlink: + # Check that writing over a file preserves a symlink + with atomic_writing(f2) as f: + f.write(u'written from symlink') + + with stdlib_io.open(f1, 'r') as f: + nt.assert_equal(f.read(), u'written from symlink') + +def _save_umask(): + global umask + umask = os.umask(0) + os.umask(umask) + +def _restore_umask(): + os.umask(umask) + +@skip_win32 +@nt.with_setup(_save_umask, _restore_umask) +def test_atomic_writing_umask(): + with TemporaryDirectory() as td: + os.umask(0o022) + f1 = os.path.join(td, '1') + with atomic_writing(f1) as f: + f.write(u'1') + mode = stat.S_IMODE(os.stat(f1).st_mode) + nt.assert_equal(mode, 0o644, '{:o} != 644'.format(mode)) + + os.umask(0o057) + f2 = os.path.join(td, '2') + with atomic_writing(f2) as f: + f.write(u'2') + mode = stat.S_IMODE(os.stat(f2).st_mode) + nt.assert_equal(mode, 0o620, '{:o} != 620'.format(mode)) + + +def test_atomic_writing_newlines(): + with TemporaryDirectory() as td: + path = os.path.join(td, 'testfile') + + lf = u'a\nb\nc\n' + plat = lf.replace(u'\n', os.linesep) + crlf = lf.replace(u'\n', u'\r\n') + + # test default + with stdlib_io.open(path, 'w') as f: + f.write(lf) + with stdlib_io.open(path, 'r', newline='') as f: + read = f.read() + nt.assert_equal(read, plat) + + # test newline=LF + with stdlib_io.open(path, 'w', newline='\n') as f: + f.write(lf) + with stdlib_io.open(path, 'r', newline='') as f: + read = f.read() + nt.assert_equal(read, lf) + + # test newline=CRLF + with atomic_writing(path, newline='\r\n') as f: + f.write(lf) + with stdlib_io.open(path, 'r', newline='') as f: + read = f.read() + nt.assert_equal(read, crlf) + + # test newline=no convert + text = u'crlf\r\ncr\rlf\n' + with atomic_writing(path, newline='') as f: + f.write(text) + with stdlib_io.open(path, 'r', newline='') as f: + read = f.read() + nt.assert_equal(read, text) diff --git a/IPython/utils/io.py b/IPython/utils/io.py index 3d236eb..6bed42e 100644 --- a/IPython/utils/io.py +++ b/IPython/utils/io.py @@ -3,33 +3,24 @@ IO related utilities. """ -#----------------------------------------------------------------------------- -# Copyright (C) 2008-2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + from __future__ import print_function from __future__ import absolute_import -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- + import codecs from contextlib import contextmanager import io import os import shutil -import stat import sys import tempfile +import warnings from .capture import CapturedIO, capture_output from .py3compat import string_types, input, PY3 -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- - class IOStream: @@ -221,87 +212,11 @@ def temp_pyfile(src, ext='.py'): f.flush() return fname, f -def _copy_metadata(src, dst): - """Copy the set of metadata we want for atomic_writing. - - Permission bits and flags. We'd like to copy file ownership as well, but we - can't do that. - """ - shutil.copymode(src, dst) - st = os.stat(src) - if hasattr(os, 'chflags') and hasattr(st, 'st_flags'): - os.chflags(dst, st.st_flags) - -@contextmanager -def atomic_writing(path, text=True, encoding='utf-8', **kwargs): - """Context manager to write to a file only if the entire write is successful. - - This works by creating a temporary file in the same directory, and renaming - it over the old file if the context is exited without an error. If other - file names are hard linked to the target file, this relationship will not be - preserved. - - On Windows, there is a small chink in the atomicity: the target file is - deleted before renaming the temporary file over it. This appears to be - unavoidable. - - Parameters - ---------- - path : str - The target file to write to. - - text : bool, optional - Whether to open the file in text mode (i.e. to write unicode). Default is - True. - - encoding : str, optional - The encoding to use for files opened in text mode. Default is UTF-8. - - **kwargs - Passed to :func:`io.open`. - """ - # realpath doesn't work on Windows: http://bugs.python.org/issue9949 - # Luckily, we only need to resolve the file itself being a symlink, not - # any of its directories, so this will suffice: - if os.path.islink(path): - path = os.path.join(os.path.dirname(path), os.readlink(path)) - - dirname, basename = os.path.split(path) - tmp_dir = tempfile.mkdtemp(prefix=basename, dir=dirname) - tmp_path = os.path.join(tmp_dir, basename) - if text: - fileobj = io.open(tmp_path, 'w', encoding=encoding, **kwargs) - else: - fileobj = io.open(tmp_path, 'wb', **kwargs) - - try: - yield fileobj - except: - fileobj.close() - shutil.rmtree(tmp_dir) - raise - - # Flush to disk - fileobj.flush() - os.fsync(fileobj.fileno()) - - # Written successfully, now rename it - fileobj.close() - - # Copy permission bits, access time, etc. - try: - _copy_metadata(path, tmp_path) - except OSError: - # e.g. the file didn't already exist. Ignore any failure to copy metadata - pass - - if os.name == 'nt' and os.path.exists(path): - # Rename over existing file doesn't work on Windows - os.remove(path) - - os.rename(tmp_path, path) - shutil.rmtree(tmp_dir) - +def atomic_writing(*args, **kwargs): + """DEPRECATED: moved to IPython.html.services.contents.fileio""" + warn("IPython.utils.io.atomic_writing has moved to IPython.html.services.contents.fileio") + from IPython.html.services.contents.fileio import atomic_writing + return atomic_writing(*args, **kwargs) def raw_print(*args, **kw): """Raw print to sys.__stdout__, otherwise identical interface to print().""" @@ -323,25 +238,9 @@ def raw_print_err(*args, **kw): rprint = raw_print rprinte = raw_print_err -def unicode_std_stream(stream='stdout'): - u"""Get a wrapper to write unicode to stdout/stderr as UTF-8. - - This ignores environment variables and default encodings, to reliably write - unicode to stdout or stderr. - :: - - unicode_std_stream().write(u'ł@e¶ŧ←') - """ - assert stream in ('stdout', 'stderr') - stream = getattr(sys, stream) - if PY3: - try: - stream_b = stream.buffer - except AttributeError: - # sys.stdout has been replaced - use it directly - return stream - else: - stream_b = stream - - return codecs.getwriter('utf-8')(stream_b) +def unicode_std_stream(stream='stdout'): + """DEPRECATED, moved to jupyter_nbconvert.utils.io""" + warn("IPython.utils.io.unicode_std_stream has moved to jupyter_nbconvert.utils.io") + from jupyter_nbconvert.utils.io import unicode_std_stream + return unicode_std_stream(stream) diff --git a/IPython/utils/tests/test_io.py b/IPython/utils/tests/test_io.py index aa00a88..04c4e9e 100644 --- a/IPython/utils/tests/test_io.py +++ b/IPython/utils/tests/test_io.py @@ -18,9 +18,7 @@ import unittest import nose.tools as nt from IPython.testing.decorators import skipif, skip_win32 -from IPython.utils.io import (Tee, capture_output, unicode_std_stream, - atomic_writing, - ) +from IPython.utils.io import Tee, capture_output from IPython.utils.py3compat import doctest_refactor_print, PY3 from IPython.utils.tempdir import TemporaryDirectory @@ -86,146 +84,4 @@ def test_capture_output(): nt.assert_equal(io.stdout, 'hi, stdout\n') nt.assert_equal(io.stderr, 'hi, stderr\n') -def test_UnicodeStdStream(): - # Test wrapping a bytes-level stdout - if PY3: - stdoutb = stdlib_io.BytesIO() - stdout = stdlib_io.TextIOWrapper(stdoutb, encoding='ascii') - else: - stdout = stdoutb = stdlib_io.BytesIO() - - orig_stdout = sys.stdout - sys.stdout = stdout - try: - sample = u"@łe¶ŧ←" - unicode_std_stream().write(sample) - - output = stdoutb.getvalue().decode('utf-8') - nt.assert_equal(output, sample) - assert not stdout.closed - finally: - sys.stdout = orig_stdout - -@skipif(not PY3, "Not applicable on Python 2") -def test_UnicodeStdStream_nowrap(): - # If we replace stdout with a StringIO, it shouldn't get wrapped. - orig_stdout = sys.stdout - sys.stdout = StringIO() - try: - nt.assert_is(unicode_std_stream(), sys.stdout) - assert not sys.stdout.closed - finally: - sys.stdout = orig_stdout - -def test_atomic_writing(): - class CustomExc(Exception): pass - - with TemporaryDirectory() as td: - f1 = os.path.join(td, 'penguin') - with stdlib_io.open(f1, 'w') as f: - f.write(u'Before') - - if os.name != 'nt': - os.chmod(f1, 0o701) - orig_mode = stat.S_IMODE(os.stat(f1).st_mode) - - f2 = os.path.join(td, 'flamingo') - try: - os.symlink(f1, f2) - have_symlink = True - except (AttributeError, NotImplementedError, OSError): - # AttributeError: Python doesn't support it - # NotImplementedError: The system doesn't support it - # OSError: The user lacks the privilege (Windows) - have_symlink = False - - with nt.assert_raises(CustomExc): - with atomic_writing(f1) as f: - f.write(u'Failing write') - raise CustomExc - - # Because of the exception, the file should not have been modified - with stdlib_io.open(f1, 'r') as f: - nt.assert_equal(f.read(), u'Before') - - with atomic_writing(f1) as f: - f.write(u'Overwritten') - - with stdlib_io.open(f1, 'r') as f: - nt.assert_equal(f.read(), u'Overwritten') - - if os.name != 'nt': - mode = stat.S_IMODE(os.stat(f1).st_mode) - nt.assert_equal(mode, orig_mode) - - if have_symlink: - # Check that writing over a file preserves a symlink - with atomic_writing(f2) as f: - f.write(u'written from symlink') - - with stdlib_io.open(f1, 'r') as f: - nt.assert_equal(f.read(), u'written from symlink') - -def _save_umask(): - global umask - umask = os.umask(0) - os.umask(umask) - -def _restore_umask(): - os.umask(umask) - -@skip_win32 -@nt.with_setup(_save_umask, _restore_umask) -def test_atomic_writing_umask(): - with TemporaryDirectory() as td: - os.umask(0o022) - f1 = os.path.join(td, '1') - with atomic_writing(f1) as f: - f.write(u'1') - mode = stat.S_IMODE(os.stat(f1).st_mode) - nt.assert_equal(mode, 0o644, '{:o} != 644'.format(mode)) - - os.umask(0o057) - f2 = os.path.join(td, '2') - with atomic_writing(f2) as f: - f.write(u'2') - mode = stat.S_IMODE(os.stat(f2).st_mode) - nt.assert_equal(mode, 0o620, '{:o} != 620'.format(mode)) - - -def test_atomic_writing_newlines(): - with TemporaryDirectory() as td: - path = os.path.join(td, 'testfile') - - lf = u'a\nb\nc\n' - plat = lf.replace(u'\n', os.linesep) - crlf = lf.replace(u'\n', u'\r\n') - - # test default - with stdlib_io.open(path, 'w') as f: - f.write(lf) - with stdlib_io.open(path, 'r', newline='') as f: - read = f.read() - nt.assert_equal(read, plat) - - # test newline=LF - with stdlib_io.open(path, 'w', newline='\n') as f: - f.write(lf) - with stdlib_io.open(path, 'r', newline='') as f: - read = f.read() - nt.assert_equal(read, lf) - - # test newline=CRLF - with atomic_writing(path, newline='\r\n') as f: - f.write(lf) - with stdlib_io.open(path, 'r', newline='') as f: - read = f.read() - nt.assert_equal(read, crlf) - - # test newline=no convert - text = u'crlf\r\ncr\rlf\n' - with atomic_writing(path, newline='') as f: - f.write(text) - with stdlib_io.open(path, 'r', newline='') as f: - read = f.read() - nt.assert_equal(read, text) + diff --git a/jupyter_client/session.py b/jupyter_client/session.py index 5f0307f..4ff1f58 100644 --- a/jupyter_client/session.py +++ b/jupyter_client/session.py @@ -49,7 +49,6 @@ from zmq.eventloop.zmqstream import ZMQStream from IPython.core.release import kernel_protocol_version from IPython.config.configurable import Configurable, LoggingConfigurable -from IPython.utils import io from IPython.utils.importstring import import_item from jupyter_client.jsonutil import extract_dates, squash_dates, date_default from IPython.utils.py3compat import (str_to_bytes, str_to_unicode, unicode_type, @@ -59,6 +58,8 @@ from IPython.utils.traitlets import (CBytes, Unicode, Bool, Any, Instance, Set, TraitError, ) from jupyter_client.adapter import adapt +from traitlets.log import get_logger + #----------------------------------------------------------------------------- # utility functions @@ -653,8 +654,9 @@ class Session(Configurable): msg = self.msg(msg_or_type, content=content, parent=parent, header=header, metadata=metadata) if not os.getpid() == self.pid: - io.rprint("WARNING: attempted to send message from fork") - io.rprint(msg) + get_logger().warn("WARNING: attempted to send message from fork\n%s", + msg + ) return buffers = [] if buffers is None else buffers if self.adapt_version: diff --git a/jupyter_nbconvert/utils/io.py b/jupyter_nbconvert/utils/io.py new file mode 100644 index 0000000..c03ec7b --- /dev/null +++ b/jupyter_nbconvert/utils/io.py @@ -0,0 +1,33 @@ +# coding: utf-8 +"""io-related utilities""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +import codecs +import sys +from IPython.utils.py3compat import PY3 + + +def unicode_std_stream(stream='stdout'): + u"""Get a wrapper to write unicode to stdout/stderr as UTF-8. + + This ignores environment variables and default encodings, to reliably write + unicode to stdout or stderr. + + :: + + unicode_std_stream().write(u'ł@e¶ŧ←') + """ + assert stream in ('stdout', 'stderr') + stream = getattr(sys, stream) + if PY3: + try: + stream_b = stream.buffer + except AttributeError: + # sys.stdout has been replaced - use it directly + return stream + else: + stream_b = stream + + return codecs.getwriter('utf-8')(stream_b) diff --git a/jupyter_nbconvert/utils/pandoc.py b/jupyter_nbconvert/utils/pandoc.py index fa320de..5832ff0 100644 --- a/jupyter_nbconvert/utils/pandoc.py +++ b/jupyter_nbconvert/utils/pandoc.py @@ -1,33 +1,20 @@ """Utility for calling pandoc""" -#----------------------------------------------------------------------------- -# Copyright (c) 2014 the IPython Development Team. -# +# Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. -# -# The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- -from __future__ import print_function +from __future__ import print_function, absolute_import -# Stdlib imports import subprocess import warnings import re from io import TextIOWrapper, BytesIO -# IPython imports from IPython.utils.py3compat import cast_bytes from IPython.utils.version import check_version from IPython.utils.process import is_cmd_found, FindCmdError from .exceptions import ConversionException -#----------------------------------------------------------------------------- -# Classes and functions -#----------------------------------------------------------------------------- _minimal_version = "1.12.1" def pandoc(source, fmt, to, extra_args=None, encoding='utf-8'): diff --git a/jupyter_nbconvert/utils/tests/test_io.py b/jupyter_nbconvert/utils/tests/test_io.py new file mode 100644 index 0000000..6c4af00 --- /dev/null +++ b/jupyter_nbconvert/utils/tests/test_io.py @@ -0,0 +1,50 @@ +# encoding: utf-8 +"""Tests for utils.io""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +import io as stdlib_io +import sys + +import nose.tools as nt + +from IPython.testing.decorators import skipif +from ..io import unicode_std_stream +from IPython.utils.py3compat import PY3 + +if PY3: + from io import StringIO +else: + from StringIO import StringIO + +def test_UnicodeStdStream(): + # Test wrapping a bytes-level stdout + if PY3: + stdoutb = stdlib_io.BytesIO() + stdout = stdlib_io.TextIOWrapper(stdoutb, encoding='ascii') + else: + stdout = stdoutb = stdlib_io.BytesIO() + + orig_stdout = sys.stdout + sys.stdout = stdout + try: + sample = u"@łe¶ŧ←" + unicode_std_stream().write(sample) + + output = stdoutb.getvalue().decode('utf-8') + nt.assert_equal(output, sample) + assert not stdout.closed + finally: + sys.stdout = orig_stdout + +@skipif(not PY3, "Not applicable on Python 2") +def test_UnicodeStdStream_nowrap(): + # If we replace stdout with a StringIO, it shouldn't get wrapped. + orig_stdout = sys.stdout + sys.stdout = StringIO() + try: + nt.assert_is(unicode_std_stream(), sys.stdout) + assert not sys.stdout.closed + finally: + sys.stdout = orig_stdout diff --git a/jupyter_nbconvert/writers/stdout.py b/jupyter_nbconvert/writers/stdout.py index b816425..6970219 100644 --- a/jupyter_nbconvert/writers/stdout.py +++ b/jupyter_nbconvert/writers/stdout.py @@ -1,24 +1,13 @@ """ Contains Stdout writer """ -#----------------------------------------------------------------------------- -#Copyright (c) 2013, the IPython Development Team. -# -#Distributed under the terms of the Modified BSD License. -# -#The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. -from IPython.utils import io +from jupyter_nbconvert.utils import io from .base import WriterBase -#----------------------------------------------------------------------------- -# Classes -#----------------------------------------------------------------------------- class StdoutWriter(WriterBase): """Consumes output from nbconvert export...() methods and writes to the