Merge branch 'fp-newkernel' into newkernel

IPython/core/completer.py

@@ -61,6 +61,7 b' used, and this module (and the readline module) are silently inactive.'
 #  the file COPYING, distributed as part of this software.
 #
 #*****************************************************************************
+from __future__ import print_function
 
 #-----------------------------------------------------------------------------
 # Imports
@@ -79,7 +80,7 b' import sys'
 
 from IPython.core.error import TryNext
 from IPython.core.prefilter import ESC_MAGIC
-from IPython.utils import generics
+from IPython.utils import generics, io
 from IPython.utils.frame import debugx
 from IPython.utils.dir2 import dir2
 
@@ -138,10 +139,61 b' def single_dir_expand(matches):'
     else:
         return matches
 
-class Bunch: pass
 
-class Completer:
-    def __init__(self,namespace=None,global_namespace=None):
+class Bunch(object): pass
+
+
+class CompletionSplitter(object):
+    """An object to split an input line in a manner similar to readline.
+
+    By having our own implementation, we can expose readline-like completion in
+    a uniform manner to all frontends.  This object only needs to be given the
+    line of text to be split and the cursor position on said line, and it
+    returns the 'word' to be completed on at the cursor after splitting the
+    entire line.
+
+    What characters are used as splitting delimiters can be controlled by
+    setting the `delims` attribute (this is a property that internally
+    automatically builds the necessary """
+
+    # Private interface
+    
+    # A string of delimiter characters.  The default value makes sense for
+    # IPython's most typical usage patterns.
+    _delims = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
+
+    # The expression (a normal string) to be compiled into a regular expression
+    # for actual splitting.  We store it as an attribute mostly for ease of
+    # debugging, since this type of code can be so tricky to debug.
+    _delim_expr = None
+
+    # The regular expression that does the actual splitting
+    _delim_re = None
+
+    def __init__(self, delims=None):
+        delims = CompletionSplitter._delims if delims is None else delims
+        self.set_delims(delims)
+
+    def set_delims(self, delims):
+        """Set the delimiters for line splitting."""
+        expr = '[' + ''.join('\\'+ c for c in delims) + ']'
+        self._delim_re = re.compile(expr)
+        self._delims = delims
+        self._delim_expr = expr
+
+    def get_delims(self):
+        """Return the string of delimiter characters."""
+        return self._delims
+
+    def split_line(self, line, cursor_pos=None):
+        """Split a line of text with a cursor at the given position.
+        """
+        l = line if cursor_pos is None else line[:cursor_pos]
+        return self._delim_re.split(l)[-1]
+
+
+class Completer(object):
+    def __init__(self, namespace=None, global_namespace=None):
         """Create a new completer for the command line.
 
         Completer([namespace,global_namespace]) -> completer instance.
@@ -291,10 +343,12 b' class IPCompleter(Completer):'
           without readline, though in that case callers must provide some extra
           information on each call about the current line."""
 
-        Completer.__init__(self,namespace,global_namespace)
+        Completer.__init__(self, namespace, global_namespace)
 
         self.magic_escape = ESC_MAGIC
 
+        self.splitter = CompletionSplitter()
+
         # Readline-dependent code
         self.use_readline = use_readline
         if use_readline:
@@ -610,16 +664,20 b' class IPCompleter(Completer):'
             
         return None
                
-    def complete(self, text, line_buffer, cursor_pos=None):
+    def complete(self, text=None, line_buffer=None, cursor_pos=None):
         """Return the state-th possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
+        Note that both the text and the line_buffer are optional, but at least
+        one of them must be given.
+
         Parameters
         ----------
-          text : string
-            Text to perform the completion on.
+          text : string, optional
+            Text to perform the completion on.  If not given, the line buffer
+            is split using the instance's CompletionSplitter object.
 
           line_buffer : string, optional
             If not given, the completer attempts to obtain the current line
@@ -631,7 +689,21 b' class IPCompleter(Completer):'
             Index of the cursor in the full line buffer.  Should be provided by
             remote frontends where kernel has no access to frontend state.
         """
+        #io.rprint('COMP1 %r %r %r' % (text, line_buffer, cursor_pos))  # dbg
+
+        # if the cursor position isn't given, the only sane assumption we can
+        # make is that it's at the end of the line (the common case)
+        if cursor_pos is None:
+            cursor_pos = len(line_buffer) if text is None else len(text)
+
+        # if text is either None or an empty string, rely on the line buffer
+        if not text:
+            text = self.splitter.split_line(line_buffer, cursor_pos)
 
+        # If no line buffer is given, assume the input text is all there was
+        if line_buffer is None:
+            line_buffer = text
+        
         magic_escape = self.magic_escape
         self.full_lbuf = line_buffer
         self.lbuf = self.full_lbuf[:cursor_pos]
@@ -639,6 +711,8 b' class IPCompleter(Completer):'
         if text.startswith('~'):
             text = os.path.expanduser(text)
 
+        #io.rprint('COMP2 %r %r %r' % (text, line_buffer, cursor_pos))  # dbg
+
         # Start with a clean slate of completions
         self.matches[:] = []
         custom_res = self.dispatch_custom_completer(text)
@@ -663,8 +737,8 b' class IPCompleter(Completer):'
         # simply collapse the dict into a list for readline, but we'd have
         # richer completion semantics in other evironments.
         self.matches = sorted(set(self.matches))
-        #from IPython.utils.io import rprint; rprint(self.matches) # dbg
-        return self.matches
+        #io.rprint('COMP TEXT, MATCHES: %r, %r' % (text, self.matches)) # dbg
+        return text, self.matches
 
     def rlcomplete(self, text, state):
         """Return the state-th possible completion for 'text'.
@@ -679,15 +753,15 b' class IPCompleter(Completer):'
 
           state : int
             Counter used by readline.
-
         """
-
-        #print "rlcomplete! '%s' %s" % (text, state) # dbg
-
         if state==0:
+
             self.full_lbuf = line_buffer = self.get_line_buffer()
             cursor_pos = self.get_endidx()
 
+            #io.rprint("\nRLCOMPLETE: %r %r %r" %
+            #          (text, line_buffer, cursor_pos) ) # dbg
+
             # if there is only a tab on a line with only whitespace, instead of
             # the mostly useless 'do you want to see all million completions'
             # message, just do the right thing and give the user his tab!
@@ -699,7 +773,7 b' class IPCompleter(Completer):'
 
             # don't apply this on 'dumb' terminals, such as emacs buffers, so
             # we don't interfere with their own tab-completion mechanism.
-            if not (self.dumb_terminal or self.full_lbuf.strip()):
+            if not (self.dumb_terminal or line_buffer.strip()):
                 self.readline.insert_text('\t')
                 sys.stdout.flush()
                 return None
@@ -719,4 +793,3 b' class IPCompleter(Completer):'
             return self.matches[state]
         except IndexError:
             return None
-

IPython/core/interactiveshell.py

@@ -1370,13 +1370,15 b' class InteractiveShell(Configurable, Magic):'
     #-------------------------------------------------------------------------
 
     def complete(self, text, line=None, cursor_pos=None):
-        """Return a sorted list of all possible completions on text.
+        """Return the completed text and a list of completions.
 
         Parameters
         ----------
 
            text : string
-             A string of text to be completed on.
+             A string of text to be completed on.  It can be given as empty and
+             instead a line/position pair are given.  In this case, the
+             completer itself will split the line like readline does.
 
            line : string, optional
              The complete line that text is part of.
@@ -1384,6 +1386,14 b' class InteractiveShell(Configurable, Magic):'
            cursor_pos : int, optional
              The position of the cursor on the input line.
 
+        Returns
+        -------
+          text : string
+            The actual text that was completed.
+
+          matches : list
+            A sorted list with all possible completions.
+
         The optional arguments allow the completion to take more context into
         account, and are part of the low-level completion API.
         
@@ -1394,23 +1404,17 b' class InteractiveShell(Configurable, Magic):'
 
         Simple usage example:
 
-        In [7]: x = 'hello'
-
-        In [8]: x
-        Out[8]: 'hello'
-
-        In [9]: print x
-        hello
+        In [1]: x = 'hello'
 
-        In [10]: _ip.complete('x.l')
-        Out[10]: ['x.ljust', 'x.lower', 'x.lstrip']
+        In [2]: _ip.complete('x.l')
+        Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
         """
 
         # Inject names into __builtin__ so we can complete on the added names.
         with self.builtin_trap:
-            return self.Completer.complete(text,line_buffer=text)
+            return self.Completer.complete(text, line, cursor_pos)
 
-    def set_custom_completer(self,completer,pos=0):
+    def set_custom_completer(self, completer, pos=0):
         """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers

IPython/core/tests/test_completer.py

@@ -6,6 +6,7 b''
 
 # stdlib
 import sys
+import unittest
 
 # third party
 import nose.tools as nt
@@ -33,3 +34,50 b' def test_protect_filename():'
     for s1, s2 in pairs:
         s1p = completer.protect_filename(s1)
         nt.assert_equals(s1p, s2)
+
+
+def check_line_split(splitter, test_specs):
+    for part1, part2, split in test_specs:
+        cursor_pos = len(part1)
+        line = part1+part2
+        out = splitter.split_line(line, cursor_pos)
+        nt.assert_equal(out, split)
+
+
+def test_line_split():
+    """Basice line splitter test with default specs."""
+    sp = completer.CompletionSplitter()
+    # The format of the test specs is: part1, part2, expected answer.  Parts 1
+    # and 2 are joined into the 'line' sent to the splitter, as if the cursor
+    # was at the end of part1.  So an empty part2 represents someone hitting
+    # tab at the end of the line, the most common case.
+    t = [('run some/scrip', '', 'some/scrip'),
+         ('run scripts/er', 'ror.py foo', 'scripts/er'),
+         ('echo $HOM', '', 'HOM'),
+         ('print sys.pa', '', 'sys.pa'),
+         ('print(sys.pa', '', 'sys.pa'),
+         ("execfile('scripts/er", '', 'scripts/er'),
+         ('a[x.', '', 'x.'),
+         ('a[x.', 'y', 'x.'),
+         ('cd "some_file/', '', 'some_file/'),
+         ]
+    check_line_split(sp, t)
+
+
+class CompletionSplitterTestCase(unittest.TestCase):
+    def setUp(self):
+        self.sp = completer.CompletionSplitter()
+
+    def test_delim_setting(self):
+        self.sp.set_delims(' ')
+        nt.assert_equal(self.sp.get_delims(), ' ')
+        nt.assert_equal(self.sp._delim_expr, '[\ ]')
+
+    def test_spaces(self):
+        """Test with only spaces as split chars."""
+        self.sp.delims = ' '
+        t = [('foo', '', 'foo'),
+             ('run foo', '', 'foo'),
+             ('run foo', 'bar', 'foo'),
+             ]
+        check_line_split(self.sp, t)

IPython/frontend/qt/console/console_widget.py

@@ -378,7 +378,14 b' class ConsoleWidget(QtGui.QWidget):'
     def reset_font(self):
         """ Sets the font to the default fixed-width font for this platform.
         """
+        # FIXME: font family and size should be configurable by the user.
+        
         if sys.platform == 'win32':
+            # Fixme: we should test whether Consolas is available and use it
+            # first if it is.  Consolas ships by default from Vista onwards,
+            # it's *vastly* more readable and prettier than Courier, and is
+            # often installed even on XP systems.  So we should first check for
+            # it, and only fallback to Courier if absolutely necessary.
             name = 'Courier'
         elif sys.platform == 'darwin':
             name = 'Monaco'

IPython/frontend/qt/console/frontend_widget.py

@@ -197,7 +197,10 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
         cursor = self._get_cursor()
         if rep['parent_header']['msg_id'] == self._complete_id and \
                 cursor.position() == self._complete_pos:
-            text = '.'.join(self._get_context())
+            # The completer tells us what text was actually used for the
+            # matching, so we must move that many characters left to apply the
+            # completions.
+            text = rep['content']['matched_text']
             cursor.movePosition(QtGui.QTextCursor.Left, n=len(text))
             self._complete_with_items(cursor, rep['content']['matches'])
 
@@ -310,14 +313,22 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
         """ Performs completion at the current cursor location.
         """
         # Decide if it makes sense to do completion
-        context = self._get_context()
-        if not context:
+
+        # We should return only if the line is empty.  Otherwise, let the
+        # kernel split the line up.
+        line = self._get_input_buffer_cursor_line()
+        if not line:
             return False
 
+        # We let the kernel split the input line, so we *always* send an empty
+        # text field.  Readline-based frontends do get a real text field which
+        # they can use.
+        text = ''
+        
         # Send the completion request to the kernel
         self._complete_id = self.kernel_manager.xreq_channel.complete(
-            '.'.join(context),                       # text 
-            self._get_input_buffer_cursor_line(),    # line 
+            text,                                      # text
+            line,                                      # line
             self._get_input_buffer_cursor_column(),  # cursor_pos
             self.input_buffer)                       # block 
         self._complete_pos = self._get_cursor().position()

IPython/utils/io.py

@@ -9,6 +9,7 b' IO related utilities.'
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
+from __future__ import print_function
 
 #-----------------------------------------------------------------------------
 # Imports
@@ -38,13 +39,13 b' class IOStream:'
         except:
             try:
                 # print handles some unicode issues which may trip a plain
-                # write() call.  Attempt to emulate write() by using a
-                # trailing comma
-                print >> self.stream, data,
+                # write() call.  Emulate write() by using an empty end
+                # argument.
+                print(data, end='', file=self.stream)
             except:
                 # if we get here, something is seriously broken.
-                print >> sys.stderr, \
-                      'ERROR - failed to write data to stream:', self.stream
+                print('ERROR - failed to write data to stream:', self.stream,
+                      file=sys.stderr)
 
     # This class used to have a writeln method, but regular files and streams
     # in Python don't have this method. We need to keep this completely
@@ -240,7 +241,7 b' class NLprinter:'
         start = kw['start']; del kw['start']
         stop = kw['stop']; del kw['stop']
         if self.depth == 0 and 'header' in kw.keys():
-            print kw['header']
+            print(kw['header'])
 
         for idx in range(start,stop):
             elem = lst[idx]
@@ -277,10 +278,17 b" def temp_pyfile(src, ext='.py'):"
     return fname, f
 
 
-def rprint(*info):
+def rprint(*args, **kw):
+    """Raw print to sys.__stdout__"""
+    
+    print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
+          file=sys.__stdout__)
+    sys.__stdout__.flush()
+
+
+def rprinte(*args, **kw):
     """Raw print to sys.__stderr__"""
     
-    for item in info:
-        print >> sys.__stderr__, item,
-    print >> sys.__stderr__
+    print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
+          file=sys.__stderr__)
     sys.__stderr__.flush()

IPython/zmq/ipkernel.py

@@ -13,6 +13,7 b' Things to do:'
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
+from __future__ import print_function
 
 # Standard library imports.
 import __builtin__
@@ -25,6 +26,7 b' import zmq'
 
 # Local imports.
 from IPython.config.configurable import Configurable
+from IPython.utils import io
 from IPython.utils.traitlets import Instance
 from completer import KernelCompleter
 from entry_point import base_launch_kernel, make_argument_parser, make_kernel, \
@@ -126,11 +128,10 b' class Kernel(Configurable):'
             assert self.reply_socket.rcvmore(), "Missing message part."
             msg = self.reply_socket.recv_json()
             omsg = Message(msg)
-            print>>sys.__stdout__
-            print>>sys.__stdout__, omsg
+            io.rprint('\n', omsg)
             handler = self.handlers.get(omsg.msg_type, None)
             if handler is None:
-                print >> sys.__stderr__, "UNKNOWN MESSAGE TYPE:", omsg
+                io.rprinte("UNKNOWN MESSAGE TYPE:", omsg)
             else:
                 handler(ident, omsg)
 
@@ -142,8 +143,8 b' class Kernel(Configurable):'
         try:
             code = parent[u'content'][u'code']
         except:
-            print>>sys.__stderr__, "Got bad msg: "
-            print>>sys.__stderr__, Message(parent)
+            io.rprinte("Got bad msg: ")
+            io.rprinte(Message(parent))
             return
         pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
         self.pub_socket.send_json(pyin_msg)
@@ -200,25 +201,27 b' class Kernel(Configurable):'
 
         # Send the reply.
         reply_msg = self.session.msg(u'execute_reply', reply_content, parent)
-        print>>sys.__stdout__, Message(reply_msg)
+        io.rprint(Message(reply_msg))
         self.reply_socket.send(ident, zmq.SNDMORE)
         self.reply_socket.send_json(reply_msg)
         if reply_msg['content']['status'] == u'error':
             self._abort_queue()
 
     def complete_request(self, ident, parent):
-        matches = {'matches' : self._complete(parent),
+        txt, matches = self._complete(parent)
+        matches = {'matches' : matches,
+                   'matched_text' : txt,
                    'status' : 'ok'}
         completion_msg = self.session.send(self.reply_socket, 'complete_reply',
                                            matches, parent, ident)
-        print >> sys.__stdout__, completion_msg
+        io.rprint(completion_msg)
 
     def object_info_request(self, ident, parent):
         context = parent['content']['oname'].split('.')
         object_info = self._object_info(context)
         msg = self.session.send(self.reply_socket, 'object_info_reply',
                                 object_info, parent, ident)
-        print >> sys.__stdout__, msg
+        io.rprint(msg)
 
     def prompt_request(self, ident, parent):
         prompt_number = self.shell.displayhook.prompt_count
@@ -228,7 +231,7 b' class Kernel(Configurable):'
                    'input_sep'     : self.shell.displayhook.input_sep}
         msg = self.session.send(self.reply_socket, 'prompt_reply',
                                 content, parent, ident)
-        print >> sys.__stdout__, msg
+        io.rprint(msg)
 
     def history_request(self, ident, parent):
         output = parent['content']['output']
@@ -238,7 +241,7 b' class Kernel(Configurable):'
         content = {'history' : hist}
         msg = self.session.send(self.reply_socket, 'history_reply',
                                 content, parent, ident)
-        print >> sys.__stdout__, msg
+        io.rprint(msg)
         
     #---------------------------------------------------------------------------
     # Protected interface
@@ -254,12 +257,11 b' class Kernel(Configurable):'
             else:
                 assert self.reply_socket.rcvmore(), "Unexpected missing message part."
                 msg = self.reply_socket.recv_json()
-            print>>sys.__stdout__, "Aborting:"
-            print>>sys.__stdout__, Message(msg)
+            io.rprint("Aborting:\n", Message(msg))
             msg_type = msg['msg_type']
             reply_type = msg_type.split('_')[0] + '_reply'
             reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg)
-            print>>sys.__stdout__, Message(reply_msg)
+            io.rprint(Message(reply_msg))
             self.reply_socket.send(ident,zmq.SNDMORE)
             self.reply_socket.send_json(reply_msg)
             # We need to wait a bit for requests to come in. This can probably
@@ -281,23 +283,22 b' class Kernel(Configurable):'
         try:
             value = reply['content']['value']
         except:
-            print>>sys.__stderr__, "Got bad raw_input reply: "
-            print>>sys.__stderr__, Message(parent)
+            io.rprinte("Got bad raw_input reply: ")
+            io.rprinte(Message(parent))
             value = ''
         return value
     
     def _complete(self, msg):
-        #from IPython.utils.io import rprint  # dbg
-        #rprint('\n\n**MSG**\n\n', msg)  # dbg
-        #import traceback; rprint(''.join(traceback.format_stack())) # dbg
         c = msg['content']
         try:
             cpos = int(c['cursor_pos'])
         except:
             # If we don't get something that we can convert to an integer, at
-            # leasat attempt the completion guessing the cursor is at the end
-            # of the text
+            # least attempt the completion guessing the cursor is at the end of
+            # the text, if there's any, and otherwise of the line
             cpos = len(c['text'])
+            if cpos==0:
+                cpos = len(c['line'])
         return self.shell.complete(c['text'], c['line'], cpos)
 
     def _object_info(self, context):