##// END OF EJS Templates
add text.getdefaultencoding() for central default encoding guess...
MinRK -
Show More
@@ -24,7 +24,7 b' import sys'
24
24
25 from IPython.external import argparse
25 from IPython.external import argparse
26 from IPython.utils.path import filefind, get_ipython_dir
26 from IPython.utils.path import filefind, get_ipython_dir
27 from IPython.utils import py3compat, warn
27 from IPython.utils import py3compat, text, warn
28
28
29 #-----------------------------------------------------------------------------
29 #-----------------------------------------------------------------------------
30 # Exceptions
30 # Exceptions
@@ -425,7 +425,7 b' class KeyValueConfigLoader(CommandLineConfigLoader):'
425 """decode argv if bytes, using stin.encoding, falling back on default enc"""
425 """decode argv if bytes, using stin.encoding, falling back on default enc"""
426 uargv = []
426 uargv = []
427 if enc is None:
427 if enc is None:
428 enc = sys.stdin.encoding or sys.getdefaultencoding()
428 enc = text.getdefaultencoding()
429 for arg in argv:
429 for arg in argv:
430 if not isinstance(arg, unicode):
430 if not isinstance(arg, unicode):
431 # only decode if not already decoded
431 # only decode if not already decoded
@@ -586,7 +586,8 b' class ArgParseConfigLoader(CommandLineConfigLoader):'
586 def _parse_args(self, args):
586 def _parse_args(self, args):
587 """self.parser->self.parsed_data"""
587 """self.parser->self.parsed_data"""
588 # decode sys.argv to support unicode command-line options
588 # decode sys.argv to support unicode command-line options
589 uargs = [py3compat.cast_unicode(a) for a in args]
589 enc = text.getdefaultencoding()
590 uargs = [py3compat.cast_unicode(a, enc) for a in args]
590 self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
591 self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
591
592
592 def _convert_to_config(self):
593 def _convert_to_config(self):
@@ -23,6 +23,7 b' from subprocess import STDOUT'
23
23
24 # our own imports
24 # our own imports
25 from ._process_common import read_no_interrupt, process_handler
25 from ._process_common import read_no_interrupt, process_handler
26 from . import text
26
27
27 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
28 # Function definitions
29 # Function definitions
@@ -88,7 +89,7 b' def _find_cmd(cmd):'
88
89
89 def _system_body(p):
90 def _system_body(p):
90 """Callback for _system."""
91 """Callback for _system."""
91 enc = sys.stdin.encoding or sys.getdefaultencoding()
92 enc = text.getdefaultencoding()
92 for line in read_no_interrupt(p.stdout).splitlines():
93 for line in read_no_interrupt(p.stdout).splitlines():
93 line = line.decode(enc, 'replace')
94 line = line.decode(enc, 'replace')
94 print(line, file=sys.stdout)
95 print(line, file=sys.stdout)
@@ -17,6 +17,7 b' import types'
17 from datetime import datetime
17 from datetime import datetime
18
18
19 from IPython.utils import py3compat
19 from IPython.utils import py3compat
20 from IPython.utils import text
20 next_attr_name = '__next__' if py3compat.PY3 else 'next'
21 next_attr_name = '__next__' if py3compat.PY3 else 'next'
21
22
22 #-----------------------------------------------------------------------------
23 #-----------------------------------------------------------------------------
@@ -134,7 +135,7 b' def json_clean(obj):'
134 return obj
135 return obj
135
136
136 if isinstance(obj, bytes):
137 if isinstance(obj, bytes):
137 return obj.decode(sys.getdefaultencoding(), 'replace')
138 return obj.decode(text.getdefaultencoding(), 'replace')
138
139
139 if isinstance(obj, container_to_list) or (
140 if isinstance(obj, container_to_list) or (
140 hasattr(obj, '__iter__') and hasattr(obj, next_attr_name)):
141 hasattr(obj, '__iter__') and hasattr(obj, next_attr_name)):
@@ -16,9 +16,11 b' Utilities for working with strings and text.'
16
16
17 import __main__
17 import __main__
18
18
19 import locale
19 import os
20 import os
20 import re
21 import re
21 import shutil
22 import shutil
23 import sys
22 import textwrap
24 import textwrap
23 from string import Formatter
25 from string import Formatter
24
26
@@ -31,6 +33,28 b' from IPython.utils.data import flatten'
31 # Code
33 # Code
32 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
33
35
36 # Less conservative replacement for sys.getdefaultencoding, that will try
37 # to match the environment.
38 # Defined here as central function, so if we find better choices, we
39 # won't need to make changes all over IPython.
40 def getdefaultencoding():
41 """Return IPython's guess for the default encoding for bytes as text.
42
43 Asks for stdin.encoding first, to match the calling Terminal, but that
44 is often None for subprocesses. Fall back on locale.getpreferredencoding()
45 which should be a sensible platform default (that respects LANG environment),
46 and finally to sys.getdefaultencoding() which is the most conservative option,
47 and usually ASCII.
48 """
49 enc = sys.stdin.encoding
50 if not enc:
51 try:
52 # There are reports of getpreferredencoding raising errors
53 # in some cases, which may well be fixed, but let's be conservative here.
54 enc = locale.getpreferredencoding(False)
55 except Exception:
56 pass
57 return enc or sys.getdefaultencoding()
34
58
35 def unquote_ends(istr):
59 def unquote_ends(istr):
36 """Remove a single pair of quotes from the endpoints of a string."""
60 """Remove a single pair of quotes from the endpoints of a string."""
@@ -4,7 +4,7 b' from io import StringIO'
4
4
5 from session import extract_header, Message
5 from session import extract_header, Message
6
6
7 from IPython.utils import io
7 from IPython.utils import io, text
8
8
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10 # Globals
10 # Globals
@@ -69,7 +69,7 b' class OutStream(object):'
69 else:
69 else:
70 # Make sure that we're handling unicode
70 # Make sure that we're handling unicode
71 if not isinstance(string, unicode):
71 if not isinstance(string, unicode):
72 enc = sys.stdin.encoding or sys.getdefaultencoding()
72 enc = text.getdefaultencoding()
73 string = string.decode(enc, 'replace')
73 string = string.decode(enc, 'replace')
74
74
75 self._buffer.write(string)
75 self._buffer.write(string)
General Comments 0
You need to be logged in to leave comments. Login now