merge from ileo-exp. Now requires trunk version of leo

IPython/Extensions/ipy_leo.py

@@ -9,15 +9,52 b' from IPython.hooks import CommandChainDispatcher'
 import re
 import UserDict
 from IPython.ipapi import TryNext 
+import IPython.macro
 
-ip = IPython.ipapi.get()
-leo = ip.user_ns['leox']
-c,g = leo.c, leo.g
+def init_ipython(ipy):
+    """ This will be run by _ip.load('ipy_leo') 
+    
+    Leo still needs to run update_commander() after this.
+    
+    """
+    global ip
+    ip = ipy
+    ip.set_hook('complete_command', mb_completer, str_key = '%mb')
+    ip.expose_magic('mb',mb_f)
+    ip.expose_magic('lee',lee_f)
+    ip.expose_magic('leoref',leoref_f)
+    expose_ileo_push(push_cl_node,100)
+    # this should be the LAST one that will be executed, and it will never raise TryNext
+    expose_ileo_push(push_ipython_script, 1000)
+    expose_ileo_push(push_plain_python, 100)
+    expose_ileo_push(push_ev_node, 100)
+    global wb
+    wb = LeoWorkbook()
+    ip.user_ns['wb'] = wb 
+    
+    show_welcome()
+
+
+def update_commander(new_leox):
+    """ Set the Leo commander to use
+    
+    This will be run every time Leo does ipython-launch; basically,
+    when the user switches the document he is focusing on, he should do
+    ipython-launch to tell ILeo what document the commands apply to.
+    
+    """
+    
+    global c,g
+    c,g = new_leox.c, new_leox.g
+    print "Set Leo Commander:",c.frame.getTitle()
 
 # will probably be overwritten by user, but handy for experimentation early on
 ip.user_ns['c'] = c
 ip.user_ns['g'] = g
+    ip.user_ns['_leo'] = new_leox
 
+    new_leox.push = push_position_from_leo
+    run_leo_startup_node()
 
 from IPython.external.simplegeneric import generic 
 import pprint
@@ -35,14 +72,44 b' def format_for_leo(obj):'
 def format_list(obj):
     return "\n".join(str(s) for s in obj)
 
+
 attribute_re = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
 def valid_attribute(s):
     return attribute_re.match(s)    
 
+_rootnode = None
+def rootnode():
+    """ Get ileo root node (@ipy-root) 
+    
+    if node has become invalid or has not been set, return None
+    
+    Note that the root is the *first* @ipy-root item found    
+    """
+    global _rootnode
+    if _rootnode is None:
+        return None
+    if c.positionExists(_rootnode.p):
+        return _rootnode
+    _rootnode = None
+    return None  
+
 def all_cells():
+    global _rootnode
     d = {}
-    for p in c.allNodes_iter():
+    r = rootnode() 
+    if r is not None:
+        nodes = r.p.children_iter()
+    else:
+        nodes = c.allNodes_iter()
+
+    for p in nodes:
         h = p.headString()
+        if h.strip() == '@ipy-root':
+            # update root node (found it for the first time)
+            _rootnode = LeoNode(p)            
+            # the next recursive call will use the children of new root
+            return all_cells()
+        
         if h.startswith('@a '):
             d[h.lstrip('@a ').strip()] = p.parent().copy()
         elif not valid_attribute(h):
@@ -50,8 +117,6 b' def all_cells():'
         d[h] = p.copy()
     return d    
     
-
-
 def eval_node(n):
     body = n.b    
     if not body.startswith('@cl'):
@@ -92,6 +157,10 b' class LeoNode(object, UserDict.DictMixin):'
     
     .ipush() - run push-to-ipython
     
+    Minibuffer command access (tab completion works):
+    
+     mb save-to-file
+    
     """
     def __init__(self,p):
         self.p = p.copy()
@@ -235,7 +304,7 b' class LeoWorkbook:'
         cells = all_cells()
         p = cells.get(key, None)
         if p is None:
-            p = add_var(key)
+            return add_var(key)
 
         return LeoNode(p)
 
@@ -260,10 +329,6 b' class LeoWorkbook:'
                 yield node
         return
             
-ip.user_ns['wb'] = LeoWorkbook()
-
-
-
 @IPython.generics.complete_object.when_type(LeoWorkbook)
 def workbook_complete(obj, prev):
     return all_cells().keys() + [s for s in prev if not s.startswith('_')]
@@ -271,17 +336,23 b' def workbook_complete(obj, prev):'
 
 def add_var(varname):
     c.beginUpdate()
+    r = rootnode()
     try:
+        if r is None:
         p2 = g.findNodeAnywhere(c,varname)
+        else:
+            p2 = g.findNodeInChildren(c, r.p, varname)
         if p2:
-            return
+            return LeoNode(p2)
+
+        if r is not None:
+            p2 = r.p.insertAsLastChild()
+        
+        else:
+            p2 =  c.currentPosition().insertAfter()
 
-        rootpos = g.findNodeAnywhere(c,'@ipy-results')
-        if not rootpos:
-            rootpos = c.currentPosition() 
-        p2 = rootpos.insertAsLastChild()
         c.setHeadString(p2,varname)
-        return p2
+        return LeoNode(p2)
     finally:
         c.endUpdate()
 
@@ -302,8 +373,9 b' def push_ipython_script(node):'
         script = node.script()
         
         script = g.splitLines(script + '\n')
-        
+        ip.user_ns['_p'] = node
         ip.runlines(script)
+        ip.user_ns.pop('_p',None)
         
         has_output = False
         for idx in range(hstart,len(ip.IP.input_hist)):
@@ -322,8 +394,6 b' def push_ipython_script(node):'
     finally:
         c.endUpdate()
 
-# this should be the LAST one that will be executed, and it will never raise TryNext
-expose_ileo_push(push_ipython_script, 1000)
     
 def eval_body(body):
     try:
@@ -345,12 +415,11 b' def push_plain_python(node):'
         raise
     es('ipy plain: %s (%d LL)' % (node.h,lines))
     
-expose_ileo_push(push_plain_python, 100)
 
 def push_cl_node(node):
     """ If node starts with @cl, eval it
     
-    The result is put to root @ipy-results node
+    The result is put as last child of @ipy-results node, if it exists
     """
     if not node.b.startswith('@cl'):
         raise TryNext
@@ -362,24 +431,80 b' def push_cl_node(node):'
         LeoNode(p2).v = val
     es(val)
 
-expose_ileo_push(push_cl_node,100)
+def push_ev_node(node):
+    """ If headline starts with @ev, eval it and put result in body """
+    if not node.h.startswith('@ev '):
+        raise TryNext
+    expr = node.h.lstrip('@ev ')
+    es('ipy eval ' + expr)
+    res = ip.ev(expr)
+    node.v = res
+    
 
 def push_position_from_leo(p):
     push_from_leo(LeoNode(p))   
     
-ip.user_ns['leox'].push = push_position_from_leo    
+@generic
+def edit_object_in_leo(obj, varname):
+    """ Make it @cl node so it can be pushed back directly by alt+I """
+    node = add_var(varname)
+    formatted = format_for_leo(obj)
+    if not formatted.startswith('@cl'):
+        formatted = '@cl\n' + formatted
+    node.b = formatted 
+    node.go()
+    
+@edit_object_in_leo.when_type(IPython.macro.Macro)
+def edit_macro(obj,varname):
+    bod = '_ip.defmacro("""\\\n' + obj.value + '""")'
+    node = add_var('Macro_' + varname)
+    node.b = bod
+    node.go()
+
+def get_history(hstart = 0):
+    res = []
+    ohist = ip.IP.output_hist 
+
+    for idx in range(hstart, len(ip.IP.input_hist)):
+        val = ohist.get(idx,None)
+        has_output = True
+        inp = ip.IP.input_hist_raw[idx]
+        if inp.strip():
+            res.append('In [%d]: %s' % (idx, inp))
+        if val:
+            res.append(pprint.pformat(val))
+            res.append('\n')    
+    return ''.join(res)
     
-def leo_f(self,s):
-    """ open file(s) in Leo
     
-    Takes an mglob pattern, e.g. '%leo *.cpp' or %leo 'rec:*.cpp'  
+def lee_f(self,s):
+    """ Open file(s)/objects in Leo
+    
+    - %lee hist -> open full session history in leo
+    - Takes an object
+    - Takes an mglob pattern, e.g. '%lee *.cpp' or %leo 'rec:*.cpp'  
     """
     import os
-    from IPython.external import mglob
     
-    files = mglob.expand(s)
     c.beginUpdate()
     try:
+        if s == 'hist':
+            wb.ipython_history.b = get_history()
+            wb.ipython_history.go()
+            return
+        
+            
+        
+        # try editing the object directly
+        obj = ip.user_ns.get(s, None)
+        if obj is not None:
+            edit_object_in_leo(obj,s)
+            return
+        
+        # if it's not object, it's a file name / mglob pattern
+        from IPython.external import mglob
+        
+        files = (os.path.abspath(f) for f in mglob.expand(s))
         for fname in files:
             p = g.findNodeAnywhere(c,'@auto ' + fname)
             if not p:
@@ -389,16 +514,17 b' def leo_f(self,s):'
             if os.path.isfile(fname):
                 c.setBodyString(p,open(fname).read())
             c.selectPosition(p)
+        print "Editing file(s), press ctrl+shift+w in Leo to write @auto nodes"
     finally:
         c.endUpdate()
 
-ip.expose_magic('leo',leo_f)
+
 
 def leoref_f(self,s):
     """ Quick reference for ILeo """
     import textwrap
     print textwrap.dedent("""\
-    %leo file - open file in leo
+    %leoe file/object - open file / object in leo
     wb.foo.v  - eval node foo (i.e. headstring is 'foo' or '@ipy foo')
     wb.foo.v = 12 - assign to body of node foo
     wb.foo.b - read or write the body of node foo
@@ -409,14 +535,15 b' def leoref_f(self,s):'
        
     """
     )
-ip.expose_magic('leoref',leoref_f)
 
-from ipy_leo import *
 
-ip = IPython.ipapi.get()
 
 def mb_f(self, arg):
-    """ Execute leo minibuffer commands """
+    """ Execute leo minibuffer commands 
+    
+    Example:
+     mb save-to-file
+    """
     c.executeMinibufferCommand(arg)
 
 def mb_completer(self,event):
@@ -430,11 +557,6 b' def mb_completer(self,event):'
     cmds.sort()
     return cmds
 
-    pass
-ip.set_hook('complete_command', mb_completer, str_key = 'mb')
-ip.expose_magic('mb',mb_f)
-
-
 def show_welcome():
     print "------------------"
     print "Welcome to Leo-enabled IPython session!"
@@ -450,6 +572,4 b' def run_leo_startup_node():'
         for n in LeoNode(p):
             push_from_leo(n)
 
-run_leo_startup_node()
-show_welcome()
 

IPython/Extensions/jobctrl.py

@@ -41,8 +41,9 b' Now launch a new IPython prompt and kill the process:'
 (you don't need to specify PID for %kill if only one task is running)
 """                     
 
-from subprocess import Popen,PIPE
+from subprocess import *
 import os,shlex,sys,time
+import threading,Queue
 
 from IPython import genutils
 
@@ -71,15 +72,70 b' def startjob(job):'
     p.line = job
     return p
 
+class AsyncJobQ(threading.Thread):
+    def __init__(self):
+        threading.Thread.__init__(self)
+        self.q = Queue.Queue()
+        self.output = []
+        self.stop = False
+    def run(self):
+        while 1:
+            cmd,cwd = self.q.get()
+            if self.stop:
+                self.output.append("** Discarding: '%s' - %s" % (cmd,cwd))
+                continue
+            self.output.append("** Task started: '%s' - %s" % (cmd,cwd))
+            
+            p = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT, cwd = cwd)
+            out = p.stdout.read()
+            self.output.append("** Task complete: '%s'\n" % cmd)
+            self.output.append(out)
+
+    def add(self,cmd):
+        self.q.put_nowait((cmd, os.getcwd()))
+        
+    def dumpoutput(self):
+        while self.output:
+            item = self.output.pop(0)
+            print item            
+
+_jobq = None
+
+def jobqueue_f(self, line):
+     
+    global _jobq
+    if not _jobq:
+        print "Starting jobqueue - do '&some_long_lasting_system_command' to enqueue"
+        _jobq = AsyncJobQ()
+        _jobq.setDaemon(True)
+        _jobq.start()
+        ip.jobq = _jobq.add
+        return
+    if line.strip() == 'stop':
+        print "Stopping and clearing jobqueue, %jobqueue start to start again"
+        _jobq.stop = True
+        return
+    if line.strip() == 'start':
+        _jobq.stop = False
+        return
+        
 def jobctrl_prefilter_f(self,line):    
     if line.startswith('&'):
         pre,fn,rest = self.split_user_input(line[1:])
         
         line = ip.IP.expand_aliases(fn,rest)
+        if not _jobq:
         return '_ip.startjob(%s)' % genutils.make_quoted_expr(line)
+        return '_ip.jobq(%s)' % genutils.make_quoted_expr(line)
 
     raise IPython.ipapi.TryNext
 
+def jobq_output_hook(self):
+    if not _jobq:
+        return
+    _jobq.dumpoutput()
+
+
 
 def job_list(ip):
     keys = ip.db.keys('tasks/*')
@@ -91,8 +147,16 b' def magic_tasks(self,line):'
 
     A 'task' is a process that has been started in IPython when 'jobctrl' extension is enabled.
     Tasks can be killed with %kill.
+    
+    '%tasks clear' clears the task list (from stale tasks)
     """
     ip = self.getapi()
+    if line.strip() == 'clear':
+        for k in ip.db.keys('tasks/*'):
+            print "Clearing",ip.db[k]
+            del ip.db[k]
+        return
+        
     ents = job_list(ip)
     if not ents:
         print "No tasks running"
@@ -133,19 +197,32 b' else:'
 
 def jobctrl_shellcmd(ip,cmd):
     """ os.system replacement that stores process info to db['tasks/t1234'] """
+    cmd = cmd.strip()
     cmdname = cmd.split(None,1)[0]
     if cmdname in shell_internal_commands:
         use_shell = True
     else:
         use_shell = False
 
+    jobentry = None
+    try:
+        try:
     p = Popen(cmd,shell = use_shell)
-    jobentry = 'tasks/t' + str(p.pid)
+        except WindowsError:
+            if use_shell:
+                # try with os.system
+                os.system(cmd)
+                return
+            else:
+                # have to go via shell, sucks
+                p = Popen(cmd,shell = True)
     
-    try:
+        jobentry = 'tasks/t' + str(p.pid)
         ip.db[jobentry] = (p.pid,cmd,os.getcwd(),time.time())
         p.communicate()
+
     finally:
+        if jobentry:
         del ip.db[jobentry]
 
 
@@ -158,5 +235,6 b' def install():'
     ip.set_hook('shell_hook', jobctrl_shellcmd)
     ip.expose_magic('kill',magic_kill)
     ip.expose_magic('tasks',magic_tasks)
-    
+    ip.expose_magic('jobqueue',jobqueue_f)
+    ip.set_hook('pre_prompt_hook', jobq_output_hook) 
 install()

IPython/Magic.py

@@ -2670,6 +2670,7 b' Defaulting color scheme to \'NoColor\'"""'
         parameter_s = parameter_s.strip()
         #bkms = self.shell.persist.get("bookmarks",{})
 
+        oldcwd = os.getcwd()
         numcd = re.match(r'(-)(\d+)$',parameter_s)
         # jump in directory history by number
         if numcd:
@@ -2719,6 +2720,7 b' Defaulting color scheme to \'NoColor\'"""'
             else:
                 cwd = os.getcwd()
                 dhist = self.shell.user_ns['_dh']
+                if oldcwd != cwd:
                 dhist.append(cwd)
                 self.db['dhist'] = compress_dhist(dhist)[-100:]
                 
@@ -2728,6 +2730,8 b' Defaulting color scheme to \'NoColor\'"""'
                 platutils.set_term_title("IPy ~")
             cwd = os.getcwd()
             dhist = self.shell.user_ns['_dh']
+            
+            if oldcwd != cwd:
             dhist.append(cwd)
             self.db['dhist'] = compress_dhist(dhist)[-100:]
         if not 'q' in opts and self.shell.user_ns['_dh']:

IPython/hooks.py

@@ -56,7 +56,7 b' from pprint import PrettyPrinter'
 __all__ = ['editor', 'fix_error_editor', 'result_display',
            'input_prefilter', 'shutdown_hook', 'late_startup_hook',
            'generate_prompt', 'generate_output_prompt','shell_hook',
-           'show_in_pager']
+           'show_in_pager','pre_prompt_hook']
 
 pformat = PrettyPrinter().pformat
 
@@ -227,8 +227,13 b' def show_in_pager(self,s):'
     # raising TryNext here will use the default paging functionality
     raise ipapi.TryNext
 
-def pre_command_hook(self,cmd):
-    """" Executed before starting to execute a command """
+def pre_prompt_hook(self):
+    """ Run before displaying the next prompt
+    
+    Use this e.g. to display output from asynchronous operations (in order 
+    to not mess up text entry)   
+    """
+    
     return None
 
 def post_command_hook(self,cmd):

IPython/iplib.py

@@ -6,7 +6,6 b' Requires Python 2.3 or newer.'
 
 This file contains all the classes and helper functions specific to IPython.
 
-$Id: iplib.py 3005 2008-02-01 16:43:34Z vivainio $
 """
 
 #*****************************************************************************
@@ -377,7 +376,10 b' class InteractiveShell(object,Magic):'
         # Get system encoding at startup time.  Certain terminals (like Emacs
         # under Win32 have it set to None, and we need to have a known valid
         # encoding to use in the raw_input() method
+        try:
         self.stdin_encoding = sys.stdin.encoding or 'ascii'
+        except AttributeError:
+            self.stdin_encoding = 'ascii'
 
         # dict of things NOT to alias (keywords, builtins and some magics)
         no_alias = {}
@@ -1744,6 +1746,7 b' want to merge them back into the new files.""" % locals()'
         # exit_now is set by a call to %Exit or %Quit
         
         while not self.exit_now:
+            self.hooks.pre_prompt_hook()
             if more:
                 try:
                     prompt = self.hooks.generate_prompt(True)