Multiple improvements to tab completion.

I refactored the API quite a bit, to retain readline compatibility but
make it more independent of readline.  There's still more to do in
cleaning up our init_readline() method, but now the completer objects
have separate rlcomplete() and complete() methods.  The former uses
the quirky readline API with a state flag, while the latter is
stateless, takes only text information, and is more suitable for GUIs
and other frontends to call programatically.

Made other minor fixes to ensure the test suite passes in full.

While all this code is a bit messy, we're getting in the direction of
the APIs we need in the long run.

IPython/core/completer.py

@@ -82,7 +82,6 b' from IPython.core.prefilter import ESC_MAGIC'
 from IPython.utils import generics
 from IPython.utils.frame import debugx
 from IPython.utils.dir2 import dir2
-import IPython.utils.rlineimpl as readline
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -107,6 +106,18 b' def protect_filename(s):'
                     for ch in s])
 
 
+def mark_dirs(matches):
+    """Mark directories in input list by appending '/' to their names."""
+    out = []
+    isdir = os.path.isdir
+    for x in matches:
+        if isdir(x):
+            out.append(x+'/')
+        else:
+            out.append(x)
+    return out
+
+
 def single_dir_expand(matches):
     "Recursively expand match lists containing a single dir."
 
@@ -250,7 +261,7 b' class IPCompleter(Completer):'
     """Extension of the completer class with IPython-specific features"""
 
     def __init__(self,shell,namespace=None,global_namespace=None,
-                 omit__names=0,alias_table=None):
+                 omit__names=0, alias_table=None, use_readline=True):
         """IPCompleter() -> completer
 
         Return a completer object suitable for use by the readline library
@@ -273,17 +284,31 b' class IPCompleter(Completer):'
         to be completed explicitly starts with one or more underscores.
 
         - If alias_table is supplied, it should be a dictionary of aliases
-        to complete. """
+        to complete.
+
+        use_readline : bool, optional
+          If true, use the readline library.  This completer can still function
+          without readline, though in that case callers must provide some extra
+          information on each call about the current line."""
 
         Completer.__init__(self,namespace,global_namespace)
 
         self.magic_escape = ESC_MAGIC
+
+        # Readline-dependent code
+        self.use_readline = use_readline
+        if use_readline:
+            import IPython.utils.rlineimpl as readline
         self.readline = readline
         delims = self.readline.get_completer_delims()
         delims = delims.replace(self.magic_escape,'')
         self.readline.set_completer_delims(delims)
         self.get_line_buffer = self.readline.get_line_buffer
         self.get_endidx = self.readline.get_endidx
+        # /end readline-dependent code
+
+        # List where completion matches will be stored
+        self.matches = []
         self.omit__names = omit__names
         self.merge_completions = shell.readline_merge_completions
         self.shell = shell.shell
@@ -311,7 +336,8 b' class IPCompleter(Completer):'
                          self.file_matches,
                          self.magic_matches,
                          self.alias_matches,
-                         self.python_func_kw_matches]
+                         self.python_func_kw_matches,
+                         ]
     
     # Code contributed by Alex Schmolck, for ipython/emacs integration
     def all_completions(self, text):
@@ -414,7 +440,8 b' class IPCompleter(Completer):'
                            protect_filename(f) for f in m0]
 
         #print 'mm',matches  # dbg
-        return single_dir_expand(matches)
+        #return single_dir_expand(matches)
+        return mark_dirs(matches)
 
     def magic_matches(self, text):
         """Match magics"""
@@ -583,50 +610,37 b' class IPCompleter(Completer):'
             
         return None
                
-    def complete(self, text, state, line_buffer=None):
-        """Return the next possible completion for 'text'.
+    def complete(self, text, line_buffer, cursor_pos=None):
+        """Return the state-th possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
-        :Keywords:
-        - line_buffer: string
-        If not given, the completer attempts to obtain the current line buffer
-        via readline.  This keyword allows clients which are requesting for
-        text completions in non-readline contexts to inform the completer of
-        the entire text.
-        """
-
-        #print '\n*** COMPLETE: <%s> (%s)' % (text,state)  # dbg
+        Parameters
+        ----------
+          text : string
+            Text to perform the completion on.
 
-        # if there is only a tab on a line with only whitespace, instead
-        # of the mostly useless 'do you want to see all million
-        # completions' message, just do the right thing and give the user
-        # his tab!  Incidentally, this enables pasting of tabbed text from
-        # an editor (as long as autoindent is off).
+          line_buffer : string, optional
+            If not given, the completer attempts to obtain the current line
+            buffer via readline.  This keyword allows clients which are
+            requesting for text completions in non-readline contexts to inform
+            the completer of the entire text.
 
-        # It should be noted that at least pyreadline still shows
-        # file completions - is there a way around it?
-        
-        # don't apply this on 'dumb' terminals, such as emacs buffers, so we
-        # don't interfere with their own tab-completion mechanism.
-        if line_buffer is None:
-            self.full_lbuf = self.get_line_buffer()
-        else:
-            self.full_lbuf = line_buffer
-            
-        if not (self.dumb_terminal or self.full_lbuf.strip()):
-            self.readline.insert_text('\t')
-            return None
+          cursor_pos : int, optional
+            Index of the cursor in the full line buffer.  Should be provided by
+            remote frontends where kernel has no access to frontend state.
+        """
         
         magic_escape = self.magic_escape
+        self.full_lbuf = line_buffer
+        self.lbuf = self.full_lbuf[:cursor_pos]
 
-        self.lbuf = self.full_lbuf[:self.get_endidx()]
-
-        try:
             if text.startswith('~'):
                 text = os.path.expanduser(text)
-            if state == 0:
+
+        # Start with a clean slate of completions
+        self.matches[:] = []
                 custom_res = self.dispatch_custom_completer(text)
                 if custom_res is not None:
                     # did custom completers produce something?
@@ -644,15 +658,65 b' class IPCompleter(Completer):'
                             self.matches = matcher(text)
                             if self.matches:
                                 break
-                    self.matches = list(set(self.matches))
+        # FIXME: we should extend our api to return a dict with completions for
+        # different types of objects.  The rlcomplete() method could then
+        # simply collapse the dict into a list for readline, but we'd have
+        # richer completion semantics in other evironments.
+        self.matches = sorted(set(self.matches))
+        #from IPython.utils.io import rprint; rprint(self.matches) # dbg
+        return self.matches
+
+    def rlcomplete(self, text, state):
+        """Return the state-th possible completion for 'text'.
+
+        This is called successively with state == 0, 1, 2, ... until it
+        returns None.  The completion should begin with 'text'.
+
+        Parameters
+        ----------
+          text : string
+            Text to perform the completion on.
+
+          state : int
+            Counter used by readline.
+
+        """
+
+        #print "rlcomplete! '%s' %s" % (text, state) # dbg
+
+        if state==0:
+            self.full_lbuf = line_buffer = self.get_line_buffer()
+            cursor_pos = self.get_endidx()
+
+            # if there is only a tab on a line with only whitespace, instead of
+            # the mostly useless 'do you want to see all million completions'
+            # message, just do the right thing and give the user his tab!
+            # Incidentally, this enables pasting of tabbed text from an editor
+            # (as long as autoindent is off).
+
+            # It should be noted that at least pyreadline still shows file
+            # completions - is there a way around it?
+
+            # don't apply this on 'dumb' terminals, such as emacs buffers, so
+            # we don't interfere with their own tab-completion mechanism.
+            if not (self.dumb_terminal or self.full_lbuf.strip()):
+                self.readline.insert_text('\t')
+                sys.stdout.flush()
+                return None
+
+            # This method computes the self.matches array
+            self.complete(text, line_buffer, cursor_pos)
+
+            # Debug version, since readline silences all exceptions making it
+            # impossible to debug any problem in the above code
+
+            ## try:
+            ##     self.complete(text, line_buffer, cursor_pos)
+            ## except:
+            ##     import traceback; traceback.print_exc()
+
             try:
-                #print "MATCH: %r" % self.matches[state] # dbg
                 return self.matches[state]
             except IndexError:
                 return None
-        except:
-            #from IPython.core.ultratb import AutoFormattedTB; # dbg
-            #tb=AutoFormattedTB('Verbose');tb() #dbg
             
-            # If completion fails, don't annoy the user.
-            return None

IPython/core/interactiveshell.py

@@ -1303,7 +1303,7 b' class InteractiveShell(Configurable, Magic):'
                 else:
                     if exception_only:
                         stb = ['An exception has occurred, use %tb to see '
-                               'the full traceback.']
+                               'the full traceback.\n']
                         stb.extend(self.InteractiveTB.get_exception_only(etype,
                                                                          value))
                     else:
@@ -1327,7 +1327,9 b' class InteractiveShell(Configurable, Magic):'
         Subclasses may override this method to put the traceback on a different
         place, like a side channel.
         """
-        self.write_err('\n'.join(stb))
+        # FIXME: this should use the proper write channels, but our test suite
+        # relies on it coming out of stdout...
+        print >> sys.stdout, self.InteractiveTB.stb2text(stb)
 
     def showsyntaxerror(self, filename=None):
         """Display the syntax error that just occurred.
@@ -1367,12 +1369,23 b' class InteractiveShell(Configurable, Magic):'
     # Things related to tab completion
     #-------------------------------------------------------------------------
 
-    def complete(self, text):
+    def complete(self, text, line=None, cursor_pos=None):
         """Return a sorted list of all possible completions on text.
 
-        Inputs:
+        Parameters
+        ----------
+
+           text : string
+             A string of text to be completed on.
 
-          - text: a string of text to be completed on.
+           line : string, optional
+             The complete line that text is part of.
+
+           cursor_pos : int, optional
+             The position of the cursor on the input line.
+
+        The optional arguments allow the completion to take more context into
+        account, and are part of the low-level completion API.
 
         This is a wrapper around the completion mechanism, similar to what
         readline does at the command line when the TAB key is hit.  By
@@ -1395,23 +1408,7 b' class InteractiveShell(Configurable, Magic):'
 
         # Inject names into __builtin__ so we can complete on the added names.
         with self.builtin_trap:
-            complete = self.Completer.complete
-            state = 0
-            # use a dict so we get unique keys, since ipyhton's multiple
-            # completers can return duplicates.  When we make 2.4 a requirement,
-            # start using sets instead, which are faster.
-            comps = {}
-            while True:
-                newcomp = complete(text,state,line_buffer=text)
-                if newcomp is None:
-                    break
-                comps[newcomp] = 1
-                state += 1
-            outcomps = comps.keys()
-            outcomps.sort()
-            #print "T:",text,"OC:",outcomps  # dbg
-            #print "vars:",self.user_ns.keys()
-            return outcomps
+            return self.Completer.complete(text,line_buffer=text)
 
     def set_custom_completer(self,completer,pos=0):
         """Adds a new custom completer function.
@@ -1425,7 +1422,7 b' class InteractiveShell(Configurable, Magic):'
 
     def set_completer(self):
         """Reset readline's completer to be our own."""
-        self.readline.set_completer(self.Completer.complete)
+        self.readline.set_completer(self.Completer.rlcomplete)
 
     def set_completer_frame(self, frame=None):
         """Set the frame of the completer."""
@@ -1497,7 +1494,7 b' class InteractiveShell(Configurable, Magic):'
                          % inputrc_name)
             
             # save this in sys so embedded copies can restore it properly
-            sys.ipcompleter = self.Completer.complete
+            sys.ipcompleter = self.Completer.rlcomplete
             self.set_completer()
 
             # Configure readline according to user's prefs

IPython/core/ultratb.py

@@ -360,6 +360,10 b' class TBTools(object):'
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
+    def stb2text(self, stb):
+        """Convert a structured traceback (a list) to a string."""
+        return '\n'.join(stb)
+
     def text(self, etype, value, tb, tb_offset=None, context=5):
         """Return formatted traceback.
 
@@ -367,7 +371,7 b' class TBTools(object):'
         """
         tb_list = self.structured_traceback(etype, value, tb,
                                             tb_offset, context)
-        return '\n'.join(tb_list)
+        return self.stb2text(tb_list)
 
     def structured_traceback(self, etype, evalue, tb, tb_offset=None,
                              context=5, mode=None):
@@ -1008,6 +1012,11 b' class FormattedTB(VerboseTB, ListTB):'
 
         VerboseTB.__init__(self,color_scheme,tb_offset,long_header,
                            call_pdb=call_pdb,include_vars=include_vars)
+
+        # Different types of tracebacks are joined with different separators to
+        # form a single string.  They are taken from this dict
+        self._join_chars = dict(Plain='', Context='\n', Verbose='\n')
+        # set_mode also sets the tb_join_char attribute
         self.set_mode(mode)
         
     def _extract_tb(self,tb):
@@ -1016,10 +1025,9 b' class FormattedTB(VerboseTB, ListTB):'
         else:
             return None
 
-    def structured_traceback(self, etype, value, tb, tb_offset=None,
-                             context=5, mode=None):
+    def structured_traceback(self, etype, value, tb, tb_offset=None, context=5):
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
-        mode = self.mode if mode is None else mode
+        mode = self.mode
         if mode in self.verbose_modes:
             # Verbose modes need a full traceback
             return VerboseTB.structured_traceback(
@@ -1035,16 +1043,9 b' class FormattedTB(VerboseTB, ListTB):'
                 self, etype, value, elist, tb_offset, context
             )
 
-    def text(self, etype, value, tb, tb_offset=None, context=5, mode=None):
-        """Return formatted traceback.
-
-        If the optional mode parameter is given, it overrides the current
-        mode."""
-
-        mode = self.mode if mode is None else mode
-        tb_list = self.structured_traceback(etype, value, tb, tb_offset,
-                                            context, mode)
-        return '\n'.join(tb_list)
+    def stb2text(self, stb):
+        """Convert a structured traceback (a list) to a string."""
+        return self.tb_join_char.join(stb)
         
 
     def set_mode(self,mode=None):
@@ -1063,6 +1064,8 b' class FormattedTB(VerboseTB, ListTB):'
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
+        # Set the join character for generating text tracebacks
+        self.tb_join_char = self._join_chars[mode]
 
     # some convenient shorcuts
     def plain(self):
@@ -1117,12 +1120,12 b' class AutoFormattedTB(FormattedTB):'
             print "\nKeyboardInterrupt"
 
     def structured_traceback(self, etype=None, value=None, tb=None,
-                             tb_offset=None, context=5, mode=None):
+                             tb_offset=None, context=5):
         if etype is None:
             etype,value,tb = sys.exc_info()
         self.tb = tb
         return FormattedTB.structured_traceback(
-            self, etype, value, tb, tb_offset, context, mode )
+            self, etype, value, tb, tb_offset, context)
 
 #---------------------------------------------------------------------------
 
@@ -1151,14 +1154,10 b' class SyntaxTB(ListTB):'
         self.last_syntax_error = None
         return e
 
-    def text(self, etype, value, tb, tb_offset=None, context=5):
-        """Return formatted traceback.
+    def stb2text(self, stb):
+        """Convert a structured traceback (a list) to a string."""
+        return ''.join(stb)
 
-        Subclasses may override this if they add extra arguments.
-        """
-        tb_list = self.structured_traceback(etype, value, tb,
-                                            tb_offset, context)
-        return ''.join(tb_list)
 
 #----------------------------------------------------------------------------
 # module testing (minimal)

IPython/frontend/qt/console/frontend_widget.py

@@ -293,8 +293,15 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
 
         # Send the completion request to the kernel
         text = '.'.join(context)
+        
+        # FIXME - Evan: we need the position of the cursor in the current input
+        # buffer.  I tried this line below but the numbers I get are bogus. -
+        # Not sure what to do.  fperez.
+        cursor_pos = self._get_cursor().position()
+        
         self._complete_id = self.kernel_manager.xreq_channel.complete(
-            text, self._get_input_buffer_cursor_line(), self.input_buffer)
+            text, self._get_input_buffer_cursor_line(), cursor_pos,
+            self.input_buffer)
         self._complete_pos = self._get_cursor().position()
         return True
 

IPython/zmq/ipkernel.py

@@ -286,7 +286,18 b' class Kernel(Configurable):'
         return value
     
     def _complete(self, msg):
-        return self.shell.complete(msg.content.line)
+        #from IPython.utils.io import rprint  # dbg
+        #rprint('\n\n**MSG**\n\n', msg)  # dbg
+        #import traceback; rprint(''.join(traceback.format_stack())) # dbg
+        c = msg['content']
+        try:
+            cpos = int(c['cursor_pos'])
+        except:
+            # If we don't get something that we can convert to an integer, at
+            # leasat attempt the completion guessing the cursor is at the end
+            # of the text
+            cpos = len(c['text'])
+        return self.shell.complete(c['text'], c['line'], cpos)
 
     def _object_info(self, context):
         symbol, leftover = self._symbol_from_context(context)

IPython/zmq/kernelmanager.py

@@ -182,7 +182,7 b' class XReqSocketChannel(ZmqSocketChannel):'
         self._queue_request(msg)
         return msg['header']['msg_id']
 
-    def complete(self, text, line, block=None):
+    def complete(self, text, line, cursor_pos, block=None):
         """Tab complete text, line, block in the kernel's namespace.
 
         Parameters
@@ -199,7 +199,7 b' class XReqSocketChannel(ZmqSocketChannel):'
         -------
         The msg_id of the message sent.
         """
-        content = dict(text=text, line=line)
+        content = dict(text=text, line=line, block=block, cursor_pos=cursor_pos)
         msg = self.session.msg('complete_request', content)
         self._queue_request(msg)
         return msg['header']['msg_id']