Tweak out the process controller class so it works interactively better

In the IPython shell launched from the console, it works pretty well now.
From the qt-console, not so much.

IPython/utils/_process_win32.py

@@ -19,6 +19,7 b' from __future__ import print_function'
 import os
 import sys
 import ctypes
+import msvcrt
 
 from ctypes import c_int, POINTER
 from ctypes.wintypes import LPCWSTR, HLOCAL
@@ -121,10 +122,10 b' def system(cmd):'
     utility is meant to be used extensively in IPython, where any return value
     would trigger :func:`sys.displayhook` calls.
     """
-    with AvoidUNCPath() as path:
-        if path is not None:
-            cmd = '"pushd %s &&"%s' % (path, cmd)
-        return process_handler(cmd, _system_body)
+    # The controller provides interactivity with both
+    # stdin and stdout
+    import _process_win32_controller
+    _process_win32_controller.system(cmd)
 
 
 def getoutput(cmd):

IPython/utils/_process_win32_controller.py

@@ -58,15 +58,20 b' LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION)'
 # Win32 API constants needed
 ERROR_HANDLE_EOF = 38
 ERROR_BROKEN_PIPE = 109
+ERROR_NO_DATA = 232
 HANDLE_FLAG_INHERIT = 0x0001
 STARTF_USESTDHANDLES = 0x0100
 CREATE_SUSPENDED = 0x0004
 CREATE_NEW_CONSOLE = 0x0010
+CREATE_NO_WINDOW = 0x08000000
 STILL_ACTIVE = 259
 WAIT_TIMEOUT = 0x0102
 WAIT_FAILED = 0xFFFFFFFF
 INFINITE = 0xFFFFFFFF
 DUPLICATE_SAME_ACCESS = 0x00000002
+ENABLE_ECHO_INPUT = 0x0004
+ENABLE_LINE_INPUT = 0x0002
+ENABLE_PROCESSED_INPUT = 0x0001
 
 # Win32 API functions needed
 GetLastError = ctypes.windll.kernel32.GetLastError
@@ -108,6 +113,18 b' WriteFile = ctypes.windll.kernel32.WriteFile'
 WriteFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPVOID]
 WriteFile.restype = BOOL
 
+GetConsoleMode = ctypes.windll.kernel32.GetConsoleMode
+GetConsoleMode.argtypes = [HANDLE, LPDWORD]
+GetConsoleMode.restype = BOOL
+
+SetConsoleMode = ctypes.windll.kernel32.SetConsoleMode
+SetConsoleMode.argtypes = [HANDLE, DWORD]
+SetConsoleMode.restype = BOOL
+
+FlushConsoleInputBuffer = ctypes.windll.kernel32.FlushConsoleInputBuffer
+FlushConsoleInputBuffer.argtypes = [HANDLE]
+FlushConsoleInputBuffer.restype = BOOL
+
 WaitForSingleObject = ctypes.windll.kernel32.WaitForSingleObject
 WaitForSingleObject.argtypes = [HANDLE, DWORD]
 WaitForSingleObject.restype = DWORD
@@ -262,7 +279,7 b' class Win32ShellCommandController(object):'
             siStartInfo.hStdOutput = c_hstdout
             siStartInfo.hStdError = c_hstderr
             siStartInfo.dwFlags = STARTF_USESTDHANDLES
-            dwCreationFlags = CREATE_SUSPENDED # | CREATE_NEW_CONSOLE
+            dwCreationFlags = CREATE_SUSPENDED | CREATE_NO_WINDOW # | CREATE_NEW_CONSOLE
 
             if not CreateProcess(None,
                     u"cmd.exe /c " + cmd,
@@ -307,8 +324,6 b' class Win32ShellCommandController(object):'
         return self
 
     def _stdin_thread(self, handle, hprocess, func, stdout_func):
-        # TODO: Use WaitForInputIdle to avoid calling func() until
-        #       an input is actually requested.
         exitCode = DWORD()
         bytesWritten = DWORD(0)
         while True:
@@ -351,6 +366,9 b' class Win32ShellCommandController(object):'
                 #print("Calling writefile")
                 if not WriteFile(handle, data, len(data),
                         ctypes.byref(bytesWritten), None):
+                    # This occurs at exit
+                    if GetLastError() == ERROR_NO_DATA:
+                        return
                     raise ctypes.WinError()
                 #print("Called writefile")
                 data = data[bytesWritten.value:]
@@ -385,20 +403,21 b' class Win32ShellCommandController(object):'
         if stdout_func == None and stdin_func == None and stderr_func == None:
             return self._run_stdio()
 
-        if stderr_func != None and self.hstderr == None:
+        if stderr_func != None and self.mergeout:
             raise RuntimeError("Shell command was initiated with "
                     "merged stdin/stdout, but a separate stderr_func "
                     "was provided to the run() method")
 
         # Create a thread for each input/output handle
+        stdin_thread = None
         threads = []
         if stdin_func:
-            threads.append(threading.Thread(target=self._stdin_thread,
+            stdin_thread = threading.Thread(target=self._stdin_thread,
                                 args=(self.hstdin, self.piProcInfo.hProcess,
-                                stdin_func, stdout_func)))
+                                stdin_func, stdout_func))
         threads.append(threading.Thread(target=self._stdout_thread,
                                     args=(self.hstdout, stdout_func)))
-        if self.hstderr != None:
+        if not self.mergeout:
             if stderr_func == None:
                 stderr_func = stdout_func
             threads.append(threading.Thread(target=self._stdout_thread,
@@ -406,6 +425,8 b' class Win32ShellCommandController(object):'
         # Start the I/O threads and the process
         if ResumeThread(self.piProcInfo.hThread) == 0xFFFFFFFF:
             raise ctypes.WinError()
+        if stdin_thread is not None:
+            stdin_thread.start()
         for thread in threads:
             thread.start()
         # Wait for the process to complete
@@ -416,29 +437,67 b' class Win32ShellCommandController(object):'
         for thread in threads:
             thread.join()
 
-    def _stdin_raw(self):
-        """Uses msvcrt.kbhit/getwch to do read stdin without blocking"""
-        if msvcrt.kbhit():
-            #s = msvcrt.getwch()
-            s = msvcrt.getwch()
-            # Key code for Enter is '\r', but need to give back '\n'
-            if s == u'\r':
-                s = u'\n'
-            return s
-        else:
-            # This should make it poll at about 100 Hz, which
-            # is hopefully good enough to be responsive but
-            # doesn't waste CPU.
-            time.sleep(0.01)
+        # Wait for the stdin thread to complete
+        if stdin_thread is not None:
+            stdin_thread.join()
+
+    def _stdin_raw_nonblock(self):
+        """Use the raw Win32 handle of sys.stdin to do non-blocking reads"""
+        # WARNING: This is experimental, and produces inconsistent results.
+        #          It's possible for the handle not to be appropriate for use
+        #          with WaitForSingleObject, among other things.
+        handle = msvcrt.get_osfhandle(sys.stdin.fileno())
+        result = WaitForSingleObject(handle, 100)
+        if result == WAIT_FAILED:
+            raise ctypes.WinError()
+        elif result == WAIT_TIMEOUT:
+            print(".", end='')
             return None
+        else:
+            data = ctypes.create_string_buffer(256)
+            bytesRead = DWORD(0)
+            print('?', end='')
+
+            if not ReadFile(handle, data, 256,
+                        ctypes.byref(bytesRead), None):
+                raise ctypes.WinError()
+            # This ensures the non-blocking works with an actual console
+            # Not checking the error, so the processing will still work with
+            # other handle types
+            FlushConsoleInputBuffer(handle)
+
+            data = data.value
+            data = data.replace('\r\n', '\n')
+            data = data.replace('\r', '\n')
+            print(repr(data) + " ", end='')
+            return data
+
+    def _stdin_raw_block(self):
+        """Use a blocking stdin read"""
+        # The big problem with the blocking read is that it doesn't
+        # exit when it's supposed to in all contexts. An extra
+        # key-press may be required to trigger the exit.
+        try:
+            data = sys.stdin.read(1)
+            data = data.replace('\r', '\n')
+            return data
+        except WindowsError as we:
+            if we.winerror == ERROR_NO_DATA:
+                # This error occurs when the pipe is closed
+                return None
+            else:
+                # Otherwise let the error propagate
+                raise we
 
     def _stdout_raw(self, s):
         """Writes the string to stdout"""
         print(s, end='', file=sys.stdout)
+        sys.stdout.flush()
 
     def _stderr_raw(self, s):
         """Writes the string to stdout"""
         print(s, end='', file=sys.stderr)
+        sys.stderr.flush()
 
     def _run_stdio(self):
         """Runs the process using the system standard I/O.
@@ -447,14 +506,27 b' class Win32ShellCommandController(object):'
                    sys.stdin object is not used. Instead,
                    msvcrt.kbhit/getwch are used asynchronously.
         """
-        if self.hstderr != None:
+        # Disable Line and Echo mode
+        #lpMode = DWORD()
+        #handle = msvcrt.get_osfhandle(sys.stdin.fileno())
+        #if GetConsoleMode(handle, ctypes.byref(lpMode)):
+        #    set_console_mode = True
+        #    if not SetConsoleMode(handle, lpMode.value &
+        #            ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT)):
+        #        raise ctypes.WinError()
+
+        if self.mergeout:
             return self.run(stdout_func = self._stdout_raw,
-                    stdin_func = self._stdin_raw,
-                    stderr_func = self._stderr_raw)
+                    stdin_func = self._stdin_raw_block)
         else:
             return self.run(stdout_func = self._stdout_raw,
-                    stdin_func = self._stdin_raw)
-                    
+                    stdin_func = self._stdin_raw_block,
+                    stderr_func = self._stderr_raw)
+
+        # Restore the previous console mode
+        #if set_console_mode:
+        #    if not SetConsoleMode(handle, lpMode.value):
+        #        raise ctypes.WinError()
 
     def __exit__(self, exc_type, exc_value, traceback):
         if self.hstdin: