Merge pull request #4849 from minrk/unicode-is-the-devil

Various unicode fixes (mostly on Windows)

IPython/core/application.py

@@ -211,9 +211,15 b' class BaseIPythonApplication(Application):'
             return crashhandler.crash_handler_lite(etype, evalue, tb)
     
     def _ipython_dir_changed(self, name, old, new):
-        if old in sys.path:
-            sys.path.remove(old)
-        sys.path.append(os.path.abspath(new))
+        str_old = py3compat.cast_bytes_py2(os.path.abspath(old),
+            sys.getfilesystemencoding()
+        )
+        if str_old in sys.path:
+            sys.path.remove(str_old)
+        str_path = py3compat.cast_bytes_py2(os.path.abspath(new),
+            sys.getfilesystemencoding()
+        )
+        sys.path.append(str_path)
         if not os.path.isdir(new):
             os.makedirs(new, mode=0o777)
         readme = os.path.join(new, 'README')

IPython/kernel/launcher.py

@@ -21,6 +21,7 b' import os'
 import sys
 from subprocess import Popen, PIPE
 
+from IPython.utils.encoding import getdefaultencoding
 from IPython.utils.py3compat import cast_bytes_py2
 
 #-----------------------------------------------------------------------------
@@ -188,12 +189,14 b' def launch_kernel(cmd, stdin=None, stdout=None, stderr=None,'
         _stderr = PIPE if stderr is None else stderr
     else:
         _stdout, _stderr = stdout, stderr
-
+    
+    encoding = getdefaultencoding(prefer_stream=False)
+    
     # Spawn a kernel.
     if sys.platform == 'win32':
-        
+        # Popen on Python 2 on Windows cannot handle unicode args or cwd
+        cmd = [ cast_bytes_py2(c, encoding) for c in cmd ]
         if cwd:
-            # Popen on Python 2 on Windows cannot handle unicode cwd.
             cwd = cast_bytes_py2(cwd, sys.getfilesystemencoding() or 'ascii')
         
         from IPython.kernel.zmq.parentpoller import ParentPollerWindows

IPython/utils/encoding.py

@@ -35,16 +35,20 b' def get_stream_enc(stream, default=None):'
 # to match the environment.
 # Defined here as central function, so if we find better choices, we
 # won't need to make changes all over IPython.
-def getdefaultencoding():
+def getdefaultencoding(prefer_stream=True):
     """Return IPython's guess for the default encoding for bytes as text.
-
-    Asks for stdin.encoding first, to match the calling Terminal, but that
-    is often None for subprocesses.  Fall back on locale.getpreferredencoding()
+    
+    If prefer_stream is True (default), asks for stdin.encoding first,
+    to match the calling Terminal, but that is often None for subprocesses.
+    
+    Then fall back on locale.getpreferredencoding(),
     which should be a sensible platform default (that respects LANG environment),
     and finally to sys.getdefaultencoding() which is the most conservative option,
-    and usually ASCII.
+    and usually ASCII on Python 2 or UTF8 on Python 3.
     """
-    enc = get_stream_enc(sys.stdin)
+    enc = None
+    if prefer_stream:
+        enc = get_stream_enc(sys.stdin)
     if not enc or enc=='ascii':
         try:
             # There are reports of getpreferredencoding raising errors

IPython/utils/submodule.py

@@ -66,7 +66,11 b' def check_submodule_status(root=None):'
     for submodule in submodules:
         if not os.path.exists(submodule):
             return 'missing'
-
+    
+    # Popen can't handle unicode cwd on Windows Python 2
+    if sys.platform == 'win32' and sys.version_info[0] < 3 \
+        and not isinstance(root, bytes):
+        root = root.encode(sys.getfilesystemencoding() or 'ascii')
     # check with git submodule status
     proc = subprocess.Popen('git submodule status',
                             stdout=subprocess.PIPE,
@@ -75,12 +79,12 b' def check_submodule_status(root=None):'
                             cwd=root,
     )
     status, _ = proc.communicate()
-    status = status.decode("ascii")
+    status = status.decode("ascii", "replace")
 
     for line in status.splitlines():
-        if status.startswith('-'):
+        if line.startswith('-'):
             return 'missing'
-        elif status.startswith('+'):
+        elif line.startswith('+'):
             return 'unclean'
 
     return 'clean'

IPython/utils/syspathcontext.py

@@ -20,6 +20,8 b' Authors:'
 
 import sys
 
+from IPython.utils.py3compat import cast_bytes_py2
+
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
@@ -28,7 +30,7 b' class appended_to_syspath(object):'
     """A context for appending a directory to sys.path for a second."""
 
     def __init__(self, dir):
-        self.dir = dir
+        self.dir = cast_bytes_py2(dir, sys.getdefaultencoding())
 
     def __enter__(self):
         if self.dir not in sys.path:
@@ -50,7 +52,7 b' class prepended_to_syspath(object):'
     """A context for prepending a directory to sys.path for a second."""
 
     def __init__(self, dir):
-        self.dir = dir
+        self.dir = cast_bytes_py2(dir, sys.getdefaultencoding())
 
     def __enter__(self):
         if self.dir not in sys.path: