From 8cff4913e012306b4d2472cc5848180c65867878 2010-08-23 07:11:02 From: Fernando Perez Date: 2010-08-23 07:11:02 Subject: [PATCH] Multiple improvements to tab completion. I refactored the API quite a bit, to retain readline compatibility but make it more independent of readline. There's still more to do in cleaning up our init_readline() method, but now the completer objects have separate rlcomplete() and complete() methods. The former uses the quirky readline API with a state flag, while the latter is stateless, takes only text information, and is more suitable for GUIs and other frontends to call programatically. Made other minor fixes to ensure the test suite passes in full. While all this code is a bit messy, we're getting in the direction of the APIs we need in the long run. --- diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 96e0366..97fcd0f 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -82,7 +82,6 @@ from IPython.core.prefilter import ESC_MAGIC from IPython.utils import generics from IPython.utils.frame import debugx from IPython.utils.dir2 import dir2 -import IPython.utils.rlineimpl as readline #----------------------------------------------------------------------------- # Globals @@ -107,6 +106,18 @@ def protect_filename(s): for ch in s]) +def mark_dirs(matches): + """Mark directories in input list by appending '/' to their names.""" + out = [] + isdir = os.path.isdir + for x in matches: + if isdir(x): + out.append(x+'/') + else: + out.append(x) + return out + + def single_dir_expand(matches): "Recursively expand match lists containing a single dir." @@ -249,8 +260,8 @@ class Completer: class IPCompleter(Completer): """Extension of the completer class with IPython-specific features""" - def __init__(self,shell,namespace=None,global_namespace=None, - omit__names=0,alias_table=None): + def __init__(self, shell, namespace=None, global_namespace=None, + omit__names=0, alias_table=None, use_readline=True): """IPCompleter() -> completer Return a completer object suitable for use by the readline library @@ -273,17 +284,31 @@ class IPCompleter(Completer): to be completed explicitly starts with one or more underscores. - If alias_table is supplied, it should be a dictionary of aliases - to complete. """ + to complete. + + use_readline : bool, optional + If true, use the readline library. This completer can still function + without readline, though in that case callers must provide some extra + information on each call about the current line.""" Completer.__init__(self,namespace,global_namespace) self.magic_escape = ESC_MAGIC - self.readline = readline - delims = self.readline.get_completer_delims() - delims = delims.replace(self.magic_escape,'') - self.readline.set_completer_delims(delims) - self.get_line_buffer = self.readline.get_line_buffer - self.get_endidx = self.readline.get_endidx + + # Readline-dependent code + self.use_readline = use_readline + if use_readline: + import IPython.utils.rlineimpl as readline + self.readline = readline + delims = self.readline.get_completer_delims() + delims = delims.replace(self.magic_escape,'') + self.readline.set_completer_delims(delims) + self.get_line_buffer = self.readline.get_line_buffer + self.get_endidx = self.readline.get_endidx + # /end readline-dependent code + + # List where completion matches will be stored + self.matches = [] self.omit__names = omit__names self.merge_completions = shell.readline_merge_completions self.shell = shell.shell @@ -311,7 +336,8 @@ class IPCompleter(Completer): self.file_matches, self.magic_matches, self.alias_matches, - self.python_func_kw_matches] + self.python_func_kw_matches, + ] # Code contributed by Alex Schmolck, for ipython/emacs integration def all_completions(self, text): @@ -414,7 +440,8 @@ class IPCompleter(Completer): protect_filename(f) for f in m0] #print 'mm',matches # dbg - return single_dir_expand(matches) + #return single_dir_expand(matches) + return mark_dirs(matches) def magic_matches(self, text): """Match magics""" @@ -583,76 +610,113 @@ class IPCompleter(Completer): return None - def complete(self, text, state, line_buffer=None): - """Return the next possible completion for 'text'. + def complete(self, text, line_buffer, cursor_pos=None): + """Return the state-th possible completion for 'text'. This is called successively with state == 0, 1, 2, ... until it returns None. The completion should begin with 'text'. - :Keywords: - - line_buffer: string - If not given, the completer attempts to obtain the current line buffer - via readline. This keyword allows clients which are requesting for - text completions in non-readline contexts to inform the completer of - the entire text. + Parameters + ---------- + text : string + Text to perform the completion on. + + line_buffer : string, optional + If not given, the completer attempts to obtain the current line + buffer via readline. This keyword allows clients which are + requesting for text completions in non-readline contexts to inform + the completer of the entire text. + + cursor_pos : int, optional + Index of the cursor in the full line buffer. Should be provided by + remote frontends where kernel has no access to frontend state. """ - #print '\n*** COMPLETE: <%s> (%s)' % (text,state) # dbg + magic_escape = self.magic_escape + self.full_lbuf = line_buffer + self.lbuf = self.full_lbuf[:cursor_pos] - # if there is only a tab on a line with only whitespace, instead - # of the mostly useless 'do you want to see all million - # completions' message, just do the right thing and give the user - # his tab! Incidentally, this enables pasting of tabbed text from - # an editor (as long as autoindent is off). + if text.startswith('~'): + text = os.path.expanduser(text) - # It should be noted that at least pyreadline still shows - # file completions - is there a way around it? - - # don't apply this on 'dumb' terminals, such as emacs buffers, so we - # don't interfere with their own tab-completion mechanism. - if line_buffer is None: - self.full_lbuf = self.get_line_buffer() + # Start with a clean slate of completions + self.matches[:] = [] + custom_res = self.dispatch_custom_completer(text) + if custom_res is not None: + # did custom completers produce something? + self.matches = custom_res else: - self.full_lbuf = line_buffer - - if not (self.dumb_terminal or self.full_lbuf.strip()): - self.readline.insert_text('\t') - return None - - magic_escape = self.magic_escape + # Extend the list of completions with the results of each + # matcher, so we return results to the user from all + # namespaces. + if self.merge_completions: + self.matches = [] + for matcher in self.matchers: + self.matches.extend(matcher(text)) + else: + for matcher in self.matchers: + self.matches = matcher(text) + if self.matches: + break + # FIXME: we should extend our api to return a dict with completions for + # different types of objects. The rlcomplete() method could then + # simply collapse the dict into a list for readline, but we'd have + # richer completion semantics in other evironments. + self.matches = sorted(set(self.matches)) + #from IPython.utils.io import rprint; rprint(self.matches) # dbg + return self.matches + + def rlcomplete(self, text, state): + """Return the state-th possible completion for 'text'. - self.lbuf = self.full_lbuf[:self.get_endidx()] + This is called successively with state == 0, 1, 2, ... until it + returns None. The completion should begin with 'text'. - try: - if text.startswith('~'): - text = os.path.expanduser(text) - if state == 0: - custom_res = self.dispatch_custom_completer(text) - if custom_res is not None: - # did custom completers produce something? - self.matches = custom_res - else: - # Extend the list of completions with the results of each - # matcher, so we return results to the user from all - # namespaces. - if self.merge_completions: - self.matches = [] - for matcher in self.matchers: - self.matches.extend(matcher(text)) - else: - for matcher in self.matchers: - self.matches = matcher(text) - if self.matches: - break - self.matches = list(set(self.matches)) - try: - #print "MATCH: %r" % self.matches[state] # dbg - return self.matches[state] - except IndexError: + Parameters + ---------- + text : string + Text to perform the completion on. + + state : int + Counter used by readline. + + """ + + #print "rlcomplete! '%s' %s" % (text, state) # dbg + + if state==0: + self.full_lbuf = line_buffer = self.get_line_buffer() + cursor_pos = self.get_endidx() + + # if there is only a tab on a line with only whitespace, instead of + # the mostly useless 'do you want to see all million completions' + # message, just do the right thing and give the user his tab! + # Incidentally, this enables pasting of tabbed text from an editor + # (as long as autoindent is off). + + # It should be noted that at least pyreadline still shows file + # completions - is there a way around it? + + # don't apply this on 'dumb' terminals, such as emacs buffers, so + # we don't interfere with their own tab-completion mechanism. + if not (self.dumb_terminal or self.full_lbuf.strip()): + self.readline.insert_text('\t') + sys.stdout.flush() return None - except: - #from IPython.core.ultratb import AutoFormattedTB; # dbg - #tb=AutoFormattedTB('Verbose');tb() #dbg - - # If completion fails, don't annoy the user. + + # This method computes the self.matches array + self.complete(text, line_buffer, cursor_pos) + + # Debug version, since readline silences all exceptions making it + # impossible to debug any problem in the above code + + ## try: + ## self.complete(text, line_buffer, cursor_pos) + ## except: + ## import traceback; traceback.print_exc() + + try: + return self.matches[state] + except IndexError: return None + diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index edf5c9c..debf617 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -1303,7 +1303,7 @@ class InteractiveShell(Configurable, Magic): else: if exception_only: stb = ['An exception has occurred, use %tb to see ' - 'the full traceback.'] + 'the full traceback.\n'] stb.extend(self.InteractiveTB.get_exception_only(etype, value)) else: @@ -1327,7 +1327,9 @@ class InteractiveShell(Configurable, Magic): Subclasses may override this method to put the traceback on a different place, like a side channel. """ - self.write_err('\n'.join(stb)) + # FIXME: this should use the proper write channels, but our test suite + # relies on it coming out of stdout... + print >> sys.stdout, self.InteractiveTB.stb2text(stb) def showsyntaxerror(self, filename=None): """Display the syntax error that just occurred. @@ -1367,13 +1369,24 @@ class InteractiveShell(Configurable, Magic): # Things related to tab completion #------------------------------------------------------------------------- - def complete(self, text): + def complete(self, text, line=None, cursor_pos=None): """Return a sorted list of all possible completions on text. - Inputs: + Parameters + ---------- + + text : string + A string of text to be completed on. - - text: a string of text to be completed on. + line : string, optional + The complete line that text is part of. + cursor_pos : int, optional + The position of the cursor on the input line. + + The optional arguments allow the completion to take more context into + account, and are part of the low-level completion API. + This is a wrapper around the completion mechanism, similar to what readline does at the command line when the TAB key is hit. By exposing it as a method, it can be used by other non-readline @@ -1395,23 +1408,7 @@ class InteractiveShell(Configurable, Magic): # Inject names into __builtin__ so we can complete on the added names. with self.builtin_trap: - complete = self.Completer.complete - state = 0 - # use a dict so we get unique keys, since ipyhton's multiple - # completers can return duplicates. When we make 2.4 a requirement, - # start using sets instead, which are faster. - comps = {} - while True: - newcomp = complete(text,state,line_buffer=text) - if newcomp is None: - break - comps[newcomp] = 1 - state += 1 - outcomps = comps.keys() - outcomps.sort() - #print "T:",text,"OC:",outcomps # dbg - #print "vars:",self.user_ns.keys() - return outcomps + return self.Completer.complete(text,line_buffer=text) def set_custom_completer(self,completer,pos=0): """Adds a new custom completer function. @@ -1425,7 +1422,7 @@ class InteractiveShell(Configurable, Magic): def set_completer(self): """Reset readline's completer to be our own.""" - self.readline.set_completer(self.Completer.complete) + self.readline.set_completer(self.Completer.rlcomplete) def set_completer_frame(self, frame=None): """Set the frame of the completer.""" @@ -1497,7 +1494,7 @@ class InteractiveShell(Configurable, Magic): % inputrc_name) # save this in sys so embedded copies can restore it properly - sys.ipcompleter = self.Completer.complete + sys.ipcompleter = self.Completer.rlcomplete self.set_completer() # Configure readline according to user's prefs diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 8a4cabd..45d0168 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -360,6 +360,10 @@ class TBTools(object): self.color_scheme_table.set_active_scheme('NoColor') self.Colors = self.color_scheme_table.active_colors + def stb2text(self, stb): + """Convert a structured traceback (a list) to a string.""" + return '\n'.join(stb) + def text(self, etype, value, tb, tb_offset=None, context=5): """Return formatted traceback. @@ -367,7 +371,7 @@ class TBTools(object): """ tb_list = self.structured_traceback(etype, value, tb, tb_offset, context) - return '\n'.join(tb_list) + return self.stb2text(tb_list) def structured_traceback(self, etype, evalue, tb, tb_offset=None, context=5, mode=None): @@ -1008,6 +1012,11 @@ class FormattedTB(VerboseTB, ListTB): VerboseTB.__init__(self,color_scheme,tb_offset,long_header, call_pdb=call_pdb,include_vars=include_vars) + + # Different types of tracebacks are joined with different separators to + # form a single string. They are taken from this dict + self._join_chars = dict(Plain='', Context='\n', Verbose='\n') + # set_mode also sets the tb_join_char attribute self.set_mode(mode) def _extract_tb(self,tb): @@ -1016,10 +1025,9 @@ class FormattedTB(VerboseTB, ListTB): else: return None - def structured_traceback(self, etype, value, tb, tb_offset=None, - context=5, mode=None): + def structured_traceback(self, etype, value, tb, tb_offset=None, context=5): tb_offset = self.tb_offset if tb_offset is None else tb_offset - mode = self.mode if mode is None else mode + mode = self.mode if mode in self.verbose_modes: # Verbose modes need a full traceback return VerboseTB.structured_traceback( @@ -1035,16 +1043,9 @@ class FormattedTB(VerboseTB, ListTB): self, etype, value, elist, tb_offset, context ) - def text(self, etype, value, tb, tb_offset=None, context=5, mode=None): - """Return formatted traceback. - - If the optional mode parameter is given, it overrides the current - mode.""" - - mode = self.mode if mode is None else mode - tb_list = self.structured_traceback(etype, value, tb, tb_offset, - context, mode) - return '\n'.join(tb_list) + def stb2text(self, stb): + """Convert a structured traceback (a list) to a string.""" + return self.tb_join_char.join(stb) def set_mode(self,mode=None): @@ -1063,6 +1064,8 @@ class FormattedTB(VerboseTB, ListTB): self.mode = mode # include variable details only in 'Verbose' mode self.include_vars = (self.mode == self.valid_modes[2]) + # Set the join character for generating text tracebacks + self.tb_join_char = self._join_chars[mode] # some convenient shorcuts def plain(self): @@ -1117,12 +1120,12 @@ class AutoFormattedTB(FormattedTB): print "\nKeyboardInterrupt" def structured_traceback(self, etype=None, value=None, tb=None, - tb_offset=None, context=5, mode=None): + tb_offset=None, context=5): if etype is None: etype,value,tb = sys.exc_info() self.tb = tb return FormattedTB.structured_traceback( - self, etype, value, tb, tb_offset, context, mode ) + self, etype, value, tb, tb_offset, context) #--------------------------------------------------------------------------- @@ -1151,14 +1154,10 @@ class SyntaxTB(ListTB): self.last_syntax_error = None return e - def text(self, etype, value, tb, tb_offset=None, context=5): - """Return formatted traceback. + def stb2text(self, stb): + """Convert a structured traceback (a list) to a string.""" + return ''.join(stb) - Subclasses may override this if they add extra arguments. - """ - tb_list = self.structured_traceback(etype, value, tb, - tb_offset, context) - return ''.join(tb_list) #---------------------------------------------------------------------------- # module testing (minimal) diff --git a/IPython/frontend/qt/console/frontend_widget.py b/IPython/frontend/qt/console/frontend_widget.py index d039e27..1521860 100644 --- a/IPython/frontend/qt/console/frontend_widget.py +++ b/IPython/frontend/qt/console/frontend_widget.py @@ -293,8 +293,15 @@ class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin): # Send the completion request to the kernel text = '.'.join(context) + + # FIXME - Evan: we need the position of the cursor in the current input + # buffer. I tried this line below but the numbers I get are bogus. - + # Not sure what to do. fperez. + cursor_pos = self._get_cursor().position() + self._complete_id = self.kernel_manager.xreq_channel.complete( - text, self._get_input_buffer_cursor_line(), self.input_buffer) + text, self._get_input_buffer_cursor_line(), cursor_pos, + self.input_buffer) self._complete_pos = self._get_cursor().position() return True diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 4839785..abf730b 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -286,7 +286,18 @@ class Kernel(Configurable): return value def _complete(self, msg): - return self.shell.complete(msg.content.line) + #from IPython.utils.io import rprint # dbg + #rprint('\n\n**MSG**\n\n', msg) # dbg + #import traceback; rprint(''.join(traceback.format_stack())) # dbg + c = msg['content'] + try: + cpos = int(c['cursor_pos']) + except: + # If we don't get something that we can convert to an integer, at + # leasat attempt the completion guessing the cursor is at the end + # of the text + cpos = len(c['text']) + return self.shell.complete(c['text'], c['line'], cpos) def _object_info(self, context): symbol, leftover = self._symbol_from_context(context) diff --git a/IPython/zmq/kernelmanager.py b/IPython/zmq/kernelmanager.py index 8670c85..929f5f9 100644 --- a/IPython/zmq/kernelmanager.py +++ b/IPython/zmq/kernelmanager.py @@ -182,7 +182,7 @@ class XReqSocketChannel(ZmqSocketChannel): self._queue_request(msg) return msg['header']['msg_id'] - def complete(self, text, line, block=None): + def complete(self, text, line, cursor_pos, block=None): """Tab complete text, line, block in the kernel's namespace. Parameters @@ -199,7 +199,7 @@ class XReqSocketChannel(ZmqSocketChannel): ------- The msg_id of the message sent. """ - content = dict(text=text, line=line) + content = dict(text=text, line=line, block=block, cursor_pos=cursor_pos) msg = self.session.msg('complete_request', content) self._queue_request(msg) return msg['header']['msg_id']