Restored major default completer functionality (cd, import, run).

These completers had been put in quarantine, I've done a substantial
amount of cleanup and restructuring of this code and put it in
core.completerlib.  There's still a lot to improve there (in
particular adding unittests) but at least now it's much more readable
and cleanly structured.

This gives us back completion for cd, import and run that 0.10 had and
which we had lost since the restructuring when the code was quarantined.

Also removed the ipy_stock_completers file, the loading of these
completers is now done in the init_completer method.

IPython/core/completerlib.py

@@ -0,0 +1,337 b''
+"""Implementations for various useful completers.
+
+These are all loaded by default by IPython.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010 The IPython Development Team.
+#
+#  Distributed under the terms of the BSD License.
+#
+#  The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+# Stdlib imports
+import glob
+import inspect
+import os
+import re
+import shlex
+import sys
+
+# Third-party imports
+from time import time
+from zipimport import zipimporter
+
+# Our own imports
+from IPython.core.error import TryNext
+
+# FIXME: this should be pulled in with the right call via the component system
+from IPython.core.ipapi import get as get_ipython
+
+#-----------------------------------------------------------------------------
+# Globals and constants
+#-----------------------------------------------------------------------------
+
+# Time in seconds after which the rootmodules will be stored permanently in the
+# ipython ip.db database (kept in the user's .ipython dir).
+TIMEOUT_STORAGE = 2
+
+# Time in seconds after which we give up
+TIMEOUT_GIVEUP = 20
+
+# Regular expression for the python import statement
+import_re = re.compile(r'.*(\.so|\.py[cod]?)$')
+
+# RE for the ipython %run command (python + ipython scripts)
+magic_run_re = re.compile(r'.*(\.ipy|\.py[w]?)$')
+
+#-----------------------------------------------------------------------------
+# Local utilities
+#-----------------------------------------------------------------------------
+
+def shlex_split(x):
+    """Helper function to split lines into segments.
+    """
+    # shlex.split raises an exception if there is a syntax error in sh syntax
+    # for example if no closing " is found. This function keeps dropping the
+    # last character of the line until shlex.split does not raise
+    # an exception. It adds end of the line to the result of shlex.split
+    #
+    # Example:
+    # %run "c:/python -> ['%run','"c:/python']
+    
+    endofline = []
+    while x != '':
+        try:
+            comps = shlex.split(x)
+            if len(endofline) >= 1:
+                comps.append(''.join(endofline))
+            return comps
+        
+        except ValueError:
+            endofline = [x[-1:]]+endofline
+            x = x[:-1]
+
+    return [''.join(endofline)]
+
+def module_list(path):
+    """
+    Return the list containing the names of the modules available in the given
+    folder.
+    """
+
+    if os.path.isdir(path):
+        folder_list = os.listdir(path)
+    elif path.endswith('.egg'):
+        try:
+            folder_list = [f for f in zipimporter(path)._files]
+        except:
+            folder_list = []
+    else:
+        folder_list = []
+
+    if not folder_list:
+        return []
+
+    # A few local constants to be used in loops below
+    isfile = os.path.isfile
+    pjoin = os.path.join
+    basename = os.path.basename
+    
+    # Now find actual path matches for packages or modules
+    folder_list = [p for p in folder_list
+                   if isfile(pjoin(path, p,'__init__.py'))
+                   or import_re.match(p) ]
+
+    return [basename(p).split('.')[0] for p in folder_list]
+
+def get_root_modules():
+    """
+    Returns a list containing the names of all the modules available in the
+    folders of the pythonpath.
+    """
+    ip = get_ipython()
+
+    if 'rootmodules' in ip.db:
+        return ip.db['rootmodules']
+    
+    t = time()
+    store = False
+    modules = list(sys.builtin_module_names)
+    for path in sys.path:
+        modules += module_list(path)        
+        if time() - t >= TIMEOUT_STORAGE and not store:
+            store = True
+            print("\nCaching the list of root modules, please wait!")
+            print("(This will only be done once - type '%rehashx' to "
+                  "reset cache!)\n")
+            sys.stdout.flush()
+        if time() - t > TIMEOUT_GIVEUP:
+            print("This is taking too long, we give up.\n")
+            ip.db['rootmodules'] = []
+            return []
+    
+    modules = set(modules)
+    if '__init__' in modules:
+        modules.remove('__init__')
+    modules = list(modules)
+    if store:
+        ip.db['rootmodules'] = modules
+    return modules
+
+
+def is_importable(module, attr, only_modules):
+    if only_modules:
+        return inspect.ismodule(getattr(module, attr))
+    else:
+        return not(attr[:2] == '__' and attr[-2:] == '__')
+
+
+def try_import(mod, only_modules=False):
+    try:
+        m = __import__(mod)
+    except:
+        return []
+    mods = mod.split('.')
+    for module in mods[1:]:
+        m = getattr(m, module)
+
+    m_is_init = hasattr(m, '__file__') and '__init__' in m.__file__
+
+    completions = []
+    if (not hasattr(m, '__file__')) or (not only_modules) or m_is_init:
+        completions.extend( [attr for attr in dir(m) if
+                             is_importable(m, attr, only_modules)])
+        
+    completions.extend(getattr(m, '__all__', []))
+    if m_is_init:
+        completions.extend(module_list(os.path.dirname(m.__file__)))
+    completions = set(completions)
+    if '__init__' in completions:
+        completions.remove('__init__')
+    return list(completions)
+
+
+#-----------------------------------------------------------------------------
+# Completion-related functions.
+#-----------------------------------------------------------------------------
+
+def quick_completer(cmd, completions):
+    """ Easily create a trivial completer for a command.
+
+    Takes either a list of completions, or all completions in string (that will
+    be split on whitespace).
+    
+    Example::
+    
+        [d:\ipython]|1> import ipy_completers
+        [d:\ipython]|2> ipy_completers.quick_completer('foo', ['bar','baz'])
+        [d:\ipython]|3> foo b<TAB>
+        bar baz
+        [d:\ipython]|3> foo ba
+    """
+    
+    if isinstance(completions, basestring):
+        completions = completions.split()
+
+    def do_complete(self, event):
+        return completions
+    
+    get_ipython().set_hook('complete_command',do_complete, str_key = cmd)
+
+
+def module_completion(line):
+    """
+    Returns a list containing the completion possibilities for an import line.
+
+    The line looks like this :
+    'import xml.d'
+    'from xml.dom import'
+    """
+
+    words = line.split(' ')
+    nwords = len(words)
+
+    # from whatever <tab> -> 'import '
+    if nwords == 3 and words[0] == 'from':
+        return ['import ']
+    
+    # 'from xy<tab>' or 'import xy<tab>'
+    if nwords < 3 and (words[0] in ['import','from']) :
+        if nwords == 1:
+            return get_root_modules()
+        mod = words[1].split('.')
+        if len(mod) < 2:
+            return get_root_modules()
+        completion_list = try_import('.'.join(mod[:-1]), True)
+        return ['.'.join(mod[:-1] + [el]) for el in completion_list]
+    
+    # 'from xyz import abc<tab>'
+    if nwords >= 3 and words[0] == 'from':
+        mod = words[1]
+        return try_import(mod)
+
+#-----------------------------------------------------------------------------
+# Completers
+#-----------------------------------------------------------------------------
+# These all have the func(self, event) signature to be used as custom
+# completers
+
+def module_completer(self,event):
+    """Give completions after user has typed 'import ...' or 'from ...'"""
+
+    # This works in all versions of python.  While 2.5 has
+    # pkgutil.walk_packages(), that particular routine is fairly dangerous,
+    # since it imports *EVERYTHING* on sys.path.  That is: a) very slow b) full
+    # of possibly problematic side effects.
+    # This search the folders in the sys.path for available modules.
+
+    return module_completion(event.line)
+
+
+def magic_run_completer(self, event):
+    """Complete files that end in .py or .ipy for the %run command.
+    """
+    comps = shlex_split(event.line)
+    relpath = (len(comps) > 1 and comps[-1] or '').strip("'\"")
+
+    #print "\nev=",event  # dbg
+    #print "rp=",relpath  # dbg
+    #print 'comps=',comps  # dbg
+
+    lglob = glob.glob
+    isdir = os.path.isdir
+    if relpath.startswith('~'):
+        relpath = os.path.expanduser(relpath)
+    dirs = [f.replace('\\','/') + "/" for f in lglob(relpath+'*') if isdir(f)]
+
+    # Find if the user has already typed the first filename, after which we
+    # should complete on all files, since after the first one other files may
+    # be arguments to the input script.
+
+    if filter(magic_run_re.match, comps):
+        pys =  [f.replace('\\','/') for f in lglob('*')]
+    else:
+        pys =  [f.replace('\\','/')
+                for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') +
+                lglob(relpath + '*.pyw')]
+    return dirs + pys
+
+
+def cd_completer(self, event):
+    """Completer function for cd, which only returns directories."""
+    ip = get_ipython()
+    relpath = event.symbol
+
+    #print(event) # dbg
+    if event.line.endswith('-b') or ' -b ' in event.line:
+        # return only bookmark completions
+        bkms = self.db.get('bookmarks', None)
+        if bkms:
+            return bkms.keys()
+        else:
+            return []
+    
+    if event.symbol == '-':
+        width_dh = str(len(str(len(ip.user_ns['_dh']) + 1)))
+        # jump in directory history by number
+        fmt = '-%0' + width_dh +'d [%s]'
+        ents = [ fmt % (i,s) for i,s in enumerate(ip.user_ns['_dh'])]
+        if len(ents) > 1:
+            return ents
+        return []
+
+    if event.symbol.startswith('--'):
+        return ["--" + os.path.basename(d) for d in ip.user_ns['_dh']]
+    
+    if relpath.startswith('~'):
+        relpath = os.path.expanduser(relpath).replace('\\','/')
+
+    found = []
+    for d in [f.replace('\\','/') + '/' for f in glob.glob(relpath+'*')
+              if os.path.isdir(f)]:
+        if ' ' in d:
+            # we don't want to deal with any of that, complex code
+            # for this is elsewhere
+            raise TryNext
+        
+        found.append( d )
+
+    if not found:
+        if os.path.isdir(relpath):
+            return [relpath]
+
+        # if no completions so far, try bookmarks
+        bks = self.db.get('bookmarks',{}).keys()
+        bkmatches = [s for s in bks if s.startswith(event.symbol)]
+        if bkmatches:
+            return bkmatches
+        
+        raise TryNext
+
+    return found

IPython/core/completer.py

@@ -80,8 +80,8 b' import sys'
 
 from IPython.core.error import TryNext
 from IPython.core.prefilter import ESC_MAGIC
-from IPython.utils import generics, io
-from IPython.utils.frame import debugx
+from IPython.utils import generics
+from IPython.utils import io
 from IPython.utils.dir2 import dir2
 
 #-----------------------------------------------------------------------------
@@ -625,11 +625,15 b' class IPCompleter(Completer):'
         if not line.strip():
             return None
 
+        # Create a little structure to pass all the relevant information about
+        # the current completion to any custom completer.
         event = Bunch()
         event.line = line
         event.symbol = text
         cmd = line.split(None,1)[0]
         event.command = cmd
+        event.text_until_cursor = self.text_until_cursor
+        
         #print "\ncustom:{%s]\n" % event # dbg
         
         # for foo etc, try also to find completer for %foo
@@ -697,7 +701,6 b' class IPCompleter(Completer):'
         if line_buffer is None:
             line_buffer = text
         
-        magic_escape = self.magic_escape
         self.line_buffer = line_buffer
         self.text_until_cursor = self.line_buffer[:cursor_pos]
         #io.rprint('\nCOMP2 %r %r %r' % (text, line_buffer, cursor_pos))  # dbg
@@ -767,16 +770,24 b' class IPCompleter(Completer):'
                 sys.stdout.flush()
                 return None
 
-            # This method computes the self.matches array
+            # Note: debugging exceptions that may occur in completion is very
+            # tricky, because readline unconditionally silences them.  So if
+            # during development you suspect a bug in the completion code, turn
+            # this flag on temporarily by uncommenting the second form (don't
+            # flip the value in the first line, as the '# dbg' marker can be
+            # automatically detected and is used elsewhere).
+            DEBUG = False
+            #DEBUG = True # dbg
+            if DEBUG:
+                try:
             self.complete(text, line_buffer, cursor_pos)
+                except:
+                    import traceback; traceback.print_exc()
+            else:
+                # The normal production version is here
 
-            # Debug version, since readline silences all exceptions making it
-            # impossible to debug any problem in the above code
-
-            ## try:
-            ##     self.complete(text, line_buffer, cursor_pos)
-            ## except:
-            ##     import traceback; traceback.print_exc()
+                # This method computes the self.matches array
+                self.complete(text, line_buffer, cursor_pos)
 
         try:
             return self.matches[state]

IPython/core/interactiveshell.py

@@ -1642,16 +1642,29 b' class InteractiveShell(Configurable, Magic):'
         (typically over the network by remote frontends).
         """
         from IPython.core.completer import IPCompleter
+        from IPython.core.completerlib import (module_completer,
+                                               magic_run_completer, cd_completer)
+        
         self.Completer = IPCompleter(self,
                                      self.user_ns,
                                      self.user_global_ns,
                                      self.readline_omit__names,
                                      self.alias_manager.alias_table,
                                      self.has_readline)
+        
+        # Add custom completers to the basic ones built into IPCompleter
         sdisp = self.strdispatchers.get('complete_command', StrDispatch())
         self.strdispatchers['complete_command'] = sdisp
         self.Completer.custom_completers = sdisp
 
+        self.set_hook('complete_command', module_completer, str_key = 'import')
+        self.set_hook('complete_command', module_completer, str_key = 'from')
+        self.set_hook('complete_command', magic_run_completer, str_key = '%run')
+        self.set_hook('complete_command', cd_completer, str_key = '%cd')
+
+        # Only configure readline if we truly are using readline.  IPython can
+        # do tab-completion over the network, in GUIs, etc, where readline
+        # itself may be absent
         if self.has_readline:
             self.set_readline_completer()