New system for interactively running scripts. After a submission by Ken

Schutte <kschutte-AT-csail.mit.edu> on the ipython-user list.

IPython/irunner.py

@@ -0,0 +1,276 b''
+#!/usr/bin/env python
+"""Module for interactively running scripts.
+
+This module implements classes for interactively running scripts written for
+any system with a prompt which can be matched by a regexp suitable for
+pexpect.  It can be used to run as if they had been typed up interactively, an
+arbitrary series of commands for the target system.
+
+The module includes classes ready for IPython (with the default prompts),
+plain Python and SAGE, but making a new one is trivial.  To see how to use it,
+simply run the module as a script:
+
+./irunner.py --help
+
+
+This is an extension of Ken Schutte <kschutte-AT-csail.mit.edu>'s script
+contributed on the ipython-user list:
+
+http://scipy.net/pipermail/ipython-user/2006-May/001705.html
+
+
+NOTES:
+
+ - This module requires pexpect, available in most linux distros, or which can
+ be downloaded from
+
+    http://pexpect.sourceforge.net
+
+ - Because pexpect only works under Unix or Windows-Cygwin, this has the same
+ limitations.  This means that it will NOT work under native windows Python.
+"""
+
+# Stdlib imports
+import optparse
+import sys
+
+# Third-party modules.
+import pexpect
+
+# Global usage strings, to avoid indentation issues when typing it below.
+USAGE = """
+Interactive script runner, type: %s
+
+runner [opts] script_name
+"""
+
+# The generic runner class
+class InteractiveRunner(object):
+    """Class to run a sequence of commands through an interactive program."""
+    
+    def __init__(self,program,prompts,args=None):
+        """Construct a runner.
+
+        Inputs:
+
+          - program: command to execute the given program.
+
+          - prompts: a list of patterns to match as valid prompts, in the
+          format used by pexpect.  This basically means that it can be either
+          a string (to be compiled as a regular expression) or a list of such
+          (it must be a true list, as pexpect does type checks).
+
+          If more than one prompt is given, the first is treated as the main
+          program prompt and the others as 'continuation' prompts, like
+          python's.  This means that blank lines in the input source are
+          ommitted when the first prompt is matched, but are NOT ommitted when
+          the continuation one matches, since this is how python signals the
+          end of multiline input interactively.
+
+        Optional inputs:
+
+          - args(None): optional list of strings to pass as arguments to the
+          child program.
+        """
+        
+        self.program = program
+        self.prompts = prompts
+        if args is None: args = []
+        self.args = args
+        
+    def run_file(self,fname,interact=False):
+        """Run the given file interactively.
+
+        Inputs:
+
+          -fname: name of the file to execute.
+
+        See the run_source docstring for the meaning of the optional
+        arguments."""
+
+        fobj = open(fname,'r')
+        try:
+            self.run_source(fobj,interact)
+        finally:
+            fobj.close()
+        
+    def run_source(self,source,interact=False):
+        """Run the given source code interactively.
+
+        Inputs:
+
+          - source: a string of code to be executed, or an open file object we
+          can iterate over.
+
+        Optional inputs:
+
+          - interact(False): if true, start to interact with the running
+          program at the end of the script.  Otherwise, just exit.
+          """
+
+        # if the source is a string, chop it up in lines so we can iterate
+        # over it just as if it were an open file.
+        if not isinstance(source,file):
+            source = source.splitlines(True)
+
+        # grab the true write method of stdout, in case anything later
+        # reassigns sys.stdout, so that we really are writing to the true
+        # stdout and not to something else.
+        write = sys.stdout.write
+
+        c = pexpect.spawn(self.program,self.args,timeout=None)
+            
+        prompts = c.compile_pattern_list(self.prompts[0])
+        prompts = c.compile_pattern_list(self.prompts)
+
+        prompt_idx = c.expect_list(prompts)
+        # Flag whether the script ends normally or not, to know whether we can
+        # do anything further with the underlying process.
+        end_normal = True
+        for cmd in source:
+            # skip blank lines for all matches to the 'main' prompt, while the
+            # secondary prompts do not
+            if prompt_idx==0 and cmd.isspace():
+                continue
+
+            write(c.after)
+            c.send(cmd)
+            try:
+                prompt_idx = c.expect_list(prompts)
+            except pexpect.EOF:
+                # this will happen if the child dies unexpectedly
+                write(c.before)
+                end_normal = False
+                break
+            write(c.before)
+        
+        if isinstance(source,file):
+            source.close()
+
+        if end_normal:
+            if interact:
+                c.send('\n')
+                print '<< Starting interactive mode >>',
+                try:
+                    c.interact()
+                except OSError:
+                    # This is what fires when the child stops.  Simply print a
+                    # newline so the system prompt is alingned.  The extra
+                    # space is there to make sure it gets printed, otherwise
+                    # OS buffering sometimes just suppresses it.
+                    write(' \n')
+                    sys.stdout.flush()
+            else:
+                c.close()
+        else:
+            if interact:
+                e="Further interaction is not possible: child process is dead."
+                print >> sys.stderr, e
+                
+    def main(self,argv=None):
+        """Run as a command-line script."""
+
+        parser = optparse.OptionParser(usage=USAGE % self.__class__.__name__)
+        newopt = parser.add_option
+        newopt('-i','--interact',action='store_true',default=False,
+               help='Interact with the program after the script is run.')
+
+        opts,args = parser.parse_args(argv)
+
+        if len(args) != 1:
+            print >> sys.stderr,"You must supply exactly one file to run."
+            sys.exit(1)
+
+        self.run_file(args[0],opts.interact)
+
+
+# Specific runners for particular programs
+class IPythonRunner(InteractiveRunner):
+    """Interactive IPython runner.
+
+    This initalizes IPython in 'nocolor' mode for simplicity.  This lets us
+    avoid having to write a regexp that matches ANSI sequences, though pexpect
+    does support them.  If anyone contributes patches for ANSI color support,
+    they will be welcome.
+
+    It also sets the prompts manually, since the prompt regexps for
+    pexpect need to be matched to the actual prompts, so user-customized
+    prompts would break this.
+    """
+    
+    def __init__(self,program = 'ipython',args=None):
+        """New runner, optionally passing the ipython command to use."""
+        
+        args0 = ['-colors','NoColor',
+                 '-pi1','In [\\#]: ',
+                 '-pi2','   .\\D.: ']
+        if args is None: args = args0
+        else: args = args0 + args
+        prompts = [r'In \[\d+\]: ',r'   \.*: ']
+        InteractiveRunner.__init__(self,program,prompts,args)
+
+
+class PythonRunner(InteractiveRunner):
+    """Interactive Python runner."""
+
+    def __init__(self,program='python',args=None):
+        """New runner, optionally passing the python command to use."""
+
+        prompts = [r'>>> ',r'\.\.\. ']
+        InteractiveRunner.__init__(self,program,prompts,args)
+
+
+class SAGERunner(InteractiveRunner):
+    """Interactive SAGE runner.
+    
+    XXX - This class is currently untested, meant for feedback from the SAGE
+    team. """
+
+    def __init__(self,program='sage',args=None):
+        """New runner, optionally passing the sage command to use."""
+        print 'XXX - This class is currently untested!!!'
+        print 'It is a placeholder,  meant for feedback from the SAGE team.'
+
+        prompts = ['sage: ',r'\s*\.\.\. ']
+        InteractiveRunner.__init__(self,program,prompts,args)
+
+# Global usage string, to avoid indentation issues if typed in a function def.
+MAIN_USAGE = """
+%prog [options] file_to_run
+
+This is an interface to the various interactive runners available in this
+module.  If you want to pass specific options to one of the runners, you need
+to first terminate the main options with a '--', and then provide the runner's
+options.  For example:
+
+irunner.py --python -- --help
+
+will pass --help to the python runner.  Similarly,
+
+irunner.py --ipython -- --log test.log script.ipy
+
+will run the script.ipy file under the IPython runner, logging all output into
+the test.log file.
+"""
+
+def main():
+    """Run as a command-line script."""
+
+    parser = optparse.OptionParser(usage=MAIN_USAGE)
+    newopt = parser.add_option
+    parser.set_defaults(mode='ipython')
+    newopt('--ipython',action='store_const',dest='mode',const='ipython',
+           help='IPython interactive runner (default).')
+    newopt('--python',action='store_const',dest='mode',const='python',
+           help='Python interactive runner.')
+    newopt('--sage',action='store_const',dest='mode',const='sage',
+           help='SAGE interactive runner - UNTESTED.')
+
+    opts,args = parser.parse_args()
+    runners = dict(ipython=IPythonRunner,
+                   python=PythonRunner,
+                   sage=SAGERunner)
+    runners[opts.mode]().main(args)
+
+if __name__ == '__main__':
+    main()

test/test_irunner.py

@@ -0,0 +1,156 b''
+#!/usr/bin/env python
+"""Test suite for the irunner module.
+
+Not the most elegant or fine-grained, but it does cover at least the bulk
+functionality."""
+
+# Global to make tests extra verbose and help debugging
+VERBOSE = True
+
+# stdlib imports
+import cStringIO as StringIO
+import unittest
+
+# IPython imports
+from IPython import irunner
+from IPython.OutputTrap import OutputTrap
+
+# Testing code begins
+class RunnerTestCase(unittest.TestCase):
+
+    def _test_runner(self,runner,source,output):
+        """Test that a given runner's input/output match."""
+        
+        log = OutputTrap(out_head='',quiet_out=True)
+        log.trap_out()
+        runner.run_source(source)
+        log.release_out()
+        out = log.summary_out()
+        # this output contains nasty \r\n lineends, and the initial ipython
+        # banner.  clean it up for comparison
+        output_l = output.split()
+        out_l = out.split()
+        mismatch  = 0
+        for n in range(len(output_l)):
+            ol1 = output_l[n].strip()
+            ol2 = out_l[n].strip()
+            if ol1 != ol2:
+                mismatch += 1
+                if VERBOSE:
+                    print '<<< line %s does not match:' % n
+                    print repr(ol1)
+                    print repr(ol2)
+                    print '>>>'
+        self.assert_(mismatch==0,'Number of mismatched lines: %s' %
+                     mismatch)
+
+    def testIPython(self):
+        """Test the IPython runner."""
+        source = """
+print 'hello, this is python'
+
+# some more code
+x=1;y=2
+x+y**2
+
+# An example of autocall functionality
+from math import *
+autocall 1
+cos pi
+autocall 0
+cos pi
+cos(pi)
+
+for i in range(5):
+    print i,
+
+print "that's all folks!"
+
+%Exit
+"""
+        output = """\
+In [1]: print 'hello, this is python'
+hello, this is python
+
+In [2]: # some more code
+
+In [3]: x=1;y=2
+
+In [4]: x+y**2
+Out[4]: 5
+
+In [5]: # An example of autocall functionality
+
+In [6]: from math import *
+
+In [7]: autocall 1
+Automatic calling is: Smart
+
+In [8]: cos pi
+------> cos(pi)
+Out[8]: -1.0
+
+In [9]: autocall 0
+Automatic calling is: OFF
+
+In [10]: cos pi
+------------------------------------------------------------
+   File "<ipython console>", line 1
+     cos pi
+          ^
+SyntaxError: invalid syntax
+
+
+In [11]: cos(pi)
+Out[11]: -1.0
+
+In [12]: for i in range(5):
+   ....:         print i,
+   ....: 
+0 1 2 3 4
+
+In [13]: print "that's all folks!"
+that's all folks!
+
+In [14]: %Exit"""
+        runner = irunner.IPythonRunner()
+        self._test_runner(runner,source,output)
+
+    def testPython(self):
+        """Test the Python runner."""
+        runner = irunner.PythonRunner()
+        source = """
+print 'hello, this is python'
+
+# some more code
+x=1;y=2
+x+y**2
+
+from math import *
+cos(pi)
+
+for i in range(5):
+    print i,
+
+print "that's all folks!"
+        """
+        output = """\
+>>> print 'hello, this is python'
+hello, this is python
+>>> # some more code
+... x=1;y=2
+>>> x+y**2
+5
+>>> from math import *
+>>> cos(pi)
+-1.0
+>>> for i in range(5):
+...     print i,
+...
+0 1 2 3 4
+>>> print "that's all folks!"
+that's all folks!"""
+        self._test_runner(runner,source,output)
+        
+if __name__ == '__main__':
+    unittest.main()

IPython/iplib.py

@@ -6,7 +6,7 b' Requires Python 2.3 or newer.'
 
 This file contains all the classes and helper functions specific to IPython.
 
-$Id: iplib.py 1329 2006-05-26 07:52:45Z fperez $
+$Id: iplib.py 1332 2006-05-30 01:41:28Z fperez $
 """
 
 #*****************************************************************************
@@ -190,7 +190,6 b' class InteractiveShell(object,Magic):'
                  user_ns = None,user_global_ns=None,banner2='',
                  custom_exceptions=((),None),embedded=False):
 
-        
         # log system
         self.logger = Logger(self,logfname='ipython_log.py',logmode='rotate')
 

IPython/usage.py

@@ -6,7 +6,7 b''
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
-# $Id: usage.py 1301 2006-05-15 17:21:55Z vivainio $
+# $Id: usage.py 1332 2006-05-30 01:41:28Z fperez $
 
 from IPython import Release
 __author__  = '%s <%s>' % Release.authors['Fernando']
@@ -313,7 +313,7 b' REGULAR OPTIONS'
               Specify  the string used for input prompts. Note that if you are
               using numbered prompts, the number is represented with a '\#' in
               the  string.  Don't forget to quote strings with spaces embedded
-              in them. Default: 'In [\#]:'.
+              in them. Default: 'In [\#]: '.
 
               Most bash-like  escapes  can  be  used  to  customize  IPython's
               prompts, as well as a few additional ones which are IPython-spe-
@@ -321,10 +321,10 b' REGULAR OPTIONS'
               Customization section of the IPython HTML/PDF manual.
 
        -prompt_in2|pi2 <string>
-              Similar  to  the  previous option, but used for the continuation
-              prompts. The special sequence '\D' is similar to '\#', but  with
-              all  digits  replaced  dots  (so  you can have your continuation
-              prompt aligned with your  input  prompt).  Default:  '    .\D.:'
+              Similar to the previous option, but used for the continuation
+              prompts. The special sequence '\D' is similar to '\#', but with
+              all digits replaced dots (so you can have your continuation
+              prompt aligned with your input prompt).  Default: ' .\D.: '
               (note three spaces at the start for alignment with 'In [\#]').
 
        -prompt_out|po <string>

doc/ChangeLog

@@ -1,3 +1,15 b''
[phone]  Fernando Perez  <[e-mail]>
+
+	* IPython/irunner.py: New module to run scripts as if manually
+	typed into an interactive environment, based on pexpect.  After a
+	submission by Ken Schutte <kschutte-AT-csail.mit.edu> on the
+	ipython-user list.  Simple unittests in the tests/ directory.
+
+	* tools/release: add Will Maier, OpenBSD port maintainer, to
+	recepients list.  We are now officially part of the OpenBSD ports:
+	http://www.openbsd.org/ports.html !  Many thanks to Will for the
+	work.
+
 2006-05-26  Fernando Perez  <[e-mail]>
 
 	* IPython/ipmaker.py (make_IPython): modify sys.argv fix (below)

doc/ipython.1

@@ -287,7 +287,7 b' inclusions, IPython will stop if it reaches 15 recursive inclusions.'
 Specify the string used for input prompts. Note that if you are using
 numbered prompts, the number is represented with a '\\#' in the
 string. Don't forget to quote strings with spaces embedded in
-them. Default: 'In [\\#]:'.
+them. Default: 'In [\\#]: '.
 .br
 .sp 1
 Most bash-like escapes can be used to customize IPython's prompts, as well as
@@ -299,7 +299,7 b' manual.'
 Similar to the previous option, but used for the continuation prompts. The
 special sequence '\\D' is similar to '\\#', but with all digits replaced dots
 (so you can have your continuation prompt aligned with your input
-prompt). Default: '   .\\D.:' (note three spaces at the start for alignment
+prompt). Default: '   .\\D.: ' (note three spaces at the start for alignment
 with 'In [\\#]').
 .TP
 .B \-prompt_out|po <string>

doc/manual_base.lyx

@@ -9512,4 +9512,13 b' Belchenko'
 <bialix-AT-ukr.net>
 \family default 
  Improvements for win32 paging system.
+\layout List
+\labelwidthstring 00.00.0000
+
+Will\SpecialChar ~
+Maier 
+\family typewriter 
+<willmaier-AT-ml1.net>
+\family default 
+ Official OpenBSD port.
 \the_end

tools/release

@@ -89,7 +89,7 b' cd $www'
 
 # Alert package maintainers
 echo "Alerting package maintainers..."
-maintainers='[e-mail] [e-mail] [e-mail] [e-mail] [e-mail]'
+maintainers='[e-mail] [e-mail] [e-mail] [e-mail] [e-mail] [e-mail]'
 #maintainers='[e-mail]'
 
 for email in $maintainers