Changes for demos and access to raw history in %macro, %save and %edit.

Very lightly tested yet... but I want it accessible to others at the SAGE
conference, so it goes in.  I'll fix it if I broke something...

IPython/Magic.py

@@ -1,7 +1,7 b''
 # -*- coding: utf-8 -*-
 """Magic functions for InteractiveShell.
 
-$Id: Magic.py 1121 2006-02-01 21:12:20Z vivainio $"""
+$Id: Magic.py 1126 2006-02-06 02:31:40Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001 Janko Hauser <[e-mail]> and
@@ -98,7 +98,7 b' license. To use profiling, please install"python2.3-profiler" from non-free.""")'
 
     def default_option(self,fn,optstr):
         """Make an entry in the options_table for fn, with value optstr"""
-        
+
         if fn not in self.lsmagic():
             error("%s is not a magic function" % fn)
         self.options_table[fn] = optstr
@@ -129,12 +129,19 b' license. To use profiling, please install"python2.3-profiler" from non-free.""")'
         out.sort()
         return out
     
-    def extract_input_slices(self,slices):
+    def extract_input_slices(self,slices,raw=False):
         """Return as a string a set of input history slices.
 
-        The set of slices is given as a list of strings (like ['1','4:8','9'],
-        since this function is for use by magic functions which get their
-        arguments as strings.
+        Inputs:
+
+          - slices: the set of slices is given as a list of strings (like
+          ['1','4:8','9'], since this function is for use by magic functions
+          which get their arguments as strings.
+
+        Optional inputs:
+
+          - raw(False): by default, the processed input is used.  If this is
+          true, the raw input history is used instead.
 
         Note that slices can be called with two notations:
 
@@ -142,6 +149,11 b' license. To use profiling, please install"python2.3-profiler" from non-free.""")'
 
         N-M -> include items N..M (closed endpoint)."""
 
+        if raw:
+            hist = self.shell.input_hist_raw
+        else:
+            hist = self.shell.input_hist
+
         cmds = []
         for chunk in slices:
             if ':' in chunk:
@@ -152,7 +164,7 b' license. To use profiling, please install"python2.3-profiler" from non-free.""")'
             else:
                 ini = int(chunk)
                 fin = ini+1
-            cmds.append(self.shell.input_hist[ini:fin])
+            cmds.append(hist[ini:fin])
         return cmds
         
     def _ofind(self,oname):
@@ -1606,7 +1618,14 b' Currently the magic system has the following functions:\\n"""'
         """Define a set of input lines as a macro for future re-execution.
 
         Usage:\\
-          %macro name n1-n2 n3-n4 ... n5 .. n6 ...
+          %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
+
+        Options:
+        
+          -r: use 'raw' input.  By default, the 'processed' history is used,
+          so that magics are loaded in their transformed version to valid
+          Python.  If this option is given, the raw input as typed as the
+          command line is used instead.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
@@ -1657,10 +1676,10 b' Currently the magic system has the following functions:\\n"""'
 
           In [60]: exec In[44:48]+In[49]"""
 
-        args = parameter_s.split()
+        opts,args = self.parse_options(parameter_s,'r')
         name,ranges = args[0], args[1:]
         #print 'rng',ranges  # dbg
-        lines = self.extract_input_slices(ranges)
+        lines = self.extract_input_slices(ranges,opts.has_key('r'))
         macro = Macro(lines)
         self.shell.user_ns.update({name:macro})
         print 'Macro `%s` created. To execute, type its name (without quotes).' % name
@@ -1671,7 +1690,14 b' Currently the magic system has the following functions:\\n"""'
         """Save a set of lines to a given filename.
 
         Usage:\\
-          %save filename n1-n2 n3-n4 ... n5 .. n6 ...
+          %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
+
+        Options:
+        
+          -r: use 'raw' input.  By default, the 'processed' history is used,
+          so that magics are loaded in their transformed version to valid
+          Python.  If this option is given, the raw input as typed as the
+          command line is used instead.
 
         This function uses the same syntax as %macro for line extraction, but
         instead of creating a macro it saves the resulting string to the
@@ -1680,7 +1706,7 b' Currently the magic system has the following functions:\\n"""'
         It adds a '.py' extension to the file if you don't do so yourself, and
         it asks for confirmation before overwriting existing files."""
 
-        args = parameter_s.split()
+        opts,args = self.parse_options(parameter_s,'r')
         fname,ranges = args[0], args[1:]
         if not fname.endswith('.py'):
             fname += '.py'
@@ -1689,7 +1715,7 b' Currently the magic system has the following functions:\\n"""'
             if ans.lower() not in ['y','yes']:
                 print 'Operation cancelled.'
                 return
-        cmds = ''.join(self.extract_input_slices(ranges))
+        cmds = ''.join(self.extract_input_slices(ranges,opts.has_key('r')))
         f = file(fname,'w')
         f.write(cmds)
         f.close()
@@ -1742,6 +1768,13 b' Currently the magic system has the following functions:\\n"""'
         it was used, regardless of how long ago (in your current session) it
         was.
 
+        -r: use 'raw' input.  This option only applies to input taken from the
+        user's history.  By default, the 'processed' history is used, so that
+        magics are loaded in their transformed version to valid Python.  If
+        this option is given, the raw input as typed as the command line is
+        used instead.  When you exit the editor, it will be executed by
+        IPython's own processor.
+        
         -x: do not execute the edited code immediately upon exit. This is
         mainly useful if you are editing programs which need to be called with
         command line arguments, which you can then do using %run.
@@ -1860,11 +1893,14 b' Currently the magic system has the following functions:\\n"""'
         # custom exceptions
         class DataIsObject(Exception): pass
 
-        opts,args = self.parse_options(parameter_s,'px')
+        opts,args = self.parse_options(parameter_s,'prx')
+        # Set a few locals from the options for convenience:
+        opts_p = opts.has_key('p')
+        opts_r = opts.has_key('r')
 
         # Default line number value
         lineno = None
-        if opts.has_key('p'):
+        if opts_p:
             args = '_%s' % last_call[0]
             if not self.shell.user_ns.has_key(args):
                 args = last_call[1]
@@ -1873,7 +1909,7 b' Currently the magic system has the following functions:\\n"""'
         # let it be clobbered by successive '-p' calls.
         try:
             last_call[0] = self.shell.outputcache.prompt_count
-            if not opts.has_key('p'):
+            if not opts_p:
                 last_call[1] = parameter_s
         except:
             pass
@@ -1887,7 +1923,7 b' Currently the magic system has the following functions:\\n"""'
             # This means that you can't edit files whose names begin with
             # numbers this way. Tough.
             ranges = args.split()
-            data = ''.join(self.extract_input_slices(ranges))
+            data = ''.join(self.extract_input_slices(ranges,opts_r))
         elif args.endswith('.py'):
             filename = make_filename(args)
             data = ''
@@ -1955,7 +1991,10 b' Currently the magic system has the following functions:\\n"""'
             print
         else:
             print 'done. Executing edited code...'
-            self.shell.safe_execfile(filename,self.shell.user_ns)
+            if opts_r:
+                self.shell.runlines(file_read(filename))
+            else:
+                self.shell.safe_execfile(filename,self.shell.user_ns)
         if use_temp:
             try:
                 return open(filename).read()

IPython/demo.py

@@ -1,9 +1,24 b''
 """Module for interactive demos using IPython.
 
-This module implements a single class, Demo, for running Python scripts
-interactively in IPython for demonstrations.  With very simple markup (a few
-tags in comments), you can control points where the script stops executing and
-returns control to IPython.
+This module implements a few classes for running Python scripts interactively
+in IPython for demonstrations.  With very simple markup (a few tags in
+comments), you can control points where the script stops executing and returns
+control to IPython.
+
+The classes are (see their docstrings for further details):
+
+ - Demo: pure python demos
+
+ - IPythonDemo: demos with input to be processed by IPython as if it had been
+ typed interactively (so magics work, as well as any other special syntax you
+ may have added via input prefilters).
+
+ - LineDemo: single-line version of the Demo class.  These demos are executed
+ one line at a time, and require no markup.
+
+ - IPythonLineDemo: IPython version of the LineDemo class (the demo is
+ executed a line at a time, but processed via IPython).
+
 
 The file is run in its own empty namespace (though you can pass it a string of
 arguments as if in a command line environment, and it will see those as
@@ -54,12 +69,12 b' copy this into a file named ex_demo.py, and try running it via:'
 
 from IPython.demo import Demo
 d = Demo('ex_demo.py')
-d()  <--- Call the d object (omit the parens if you have autocall on).
+d()  <--- Call the d object (omit the parens if you have autocall set to 2).
 
 Each time you call the demo object, it runs the next block.  The demo object
-has a few useful methods for navigation, like again(), jump(), seek() and
-back().  It can be reset for a new run via reset() or reloaded from disk (in
-case you've edited the source) via reload().  See their docstrings below.
+has a few useful methods for navigation, like again(), edit(), jump(), seek()
+and back().  It can be reset for a new run via reset() or reloaded from disk
+(in case you've edited the source) via reload().  See their docstrings below.
 
 #################### EXAMPLE DEMO <ex_demo.py> ###############################
 '''A simple interactive demo to illustrate the use of IPython's Demo class.'''
@@ -95,9 +110,6 b" print 'z is now:', z"
 
 print 'bye!'
 ################### END EXAMPLE DEMO <ex_demo.py> ############################
-
-WARNING: this module uses Python 2.3 features, so it won't work in 2.2
-environments.
 """
 #*****************************************************************************
 #     Copyright (C) 2005-2006 Fernando Perez. <[e-mail]>
@@ -108,13 +120,14 b' environments.'
 #*****************************************************************************
 
 import exceptions
+import os
 import re
 import sys
 
 from IPython.PyColorize import Parser
-from IPython.genutils import marquee, shlex_split, file_read
+from IPython.genutils import marquee, shlex_split, file_read, file_readlines
 
-__all__ = ['Demo','DemoError']
+__all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']
 
 class DemoError(exceptions.Exception): pass
 
@@ -150,7 +163,7 b' class Demo:'
           can be changed at runtime simply by reassigning it to a boolean
           value.
           """
-
+        
         self.fname    = fname
         self.sys_argv = [fname] + shlex_split(arg_str)
         self.auto_all = auto_all
@@ -159,9 +172,11 b' class Demo:'
         # it ensures that things like color scheme and the like are always in
         # sync with the ipython mode being used.  This class is only meant to
         # be used inside ipython anyways,  so it's OK.
-        self.ip_showtb   = __IPYTHON__.showtraceback
         self.ip_ns       = __IPYTHON__.user_ns
         self.ip_colorize = __IPYTHON__.pycolorize
+        self.ip_showtb   = __IPYTHON__.showtraceback
+        self.ip_runlines = __IPYTHON__.runlines
+        self.shell       = __IPYTHON__
 
         # load user data and initialize data structures
         self.reload()
@@ -211,6 +226,20 b' class Demo:'
         if index<0 or index>=self.nblocks:
             raise ValueError('invalid block index %s' % index)
 
+    def _get_index(self,index):
+        """Get the current block index, validating and checking status.
+
+        Returns None if the demo is finished"""
+        
+        if index is None:
+            if self.finished:
+                print 'Demo finished.  Use reset() if you want to rerun it.'
+                return None
+            index = self.block_index
+        else:
+            self._validate_index(index)
+        return index
+
     def seek(self,index):
         """Move the current seek pointer to the given block"""
         self._validate_index(index)
@@ -230,15 +259,42 b' class Demo:'
         self.back(1)
         self()
 
+    def edit(self,index=None):
+        """Edit a block.
+
+        If no number is given, use the last block executed.
+
+        This edits the in-memory copy of the demo, it does NOT modify the
+        original source file.  If you want to do that, simply open the file in
+        an editor and use reload() when you make changes to the file.  This
+        method is meant to let you change a block during a demonstration for
+        explanatory purposes, without damaging your original script."""
+
+        index = self._get_index(index)
+        if index is None:
+            return
+        # decrease the index by one (unless we're at the very beginning), so
+        # that the default demo.edit() call opens up the sblock we've last run
+        if index>0:
+            index -= 1
+            
+        filename = self.shell.mktempfile(self.src_blocks[index])
+        self.shell.hooks.editor(filename,1)
+        new_block = file_read(filename)
+        # update the source and colored block
+        self.src_blocks[index] = new_block
+        self.src_blocks_colored[index] = self.ip_colorize(new_block)
+        self.block_index = index
+        # call to run with the newly edited index
+        self()
+        
     def show(self,index=None):
         """Show a single block on screen"""
+
+        index = self._get_index(index)
         if index is None:
-            if self.finished:
-                print 'Demo finished.  Use reset() if you want to rerun it.'
-                return
-            index = self.block_index
-        else:
-            self._validate_index(index)
+            return
+
         print marquee('<%s> block # %s (%s remaining)' %
                       (self.fname,index,self.nblocks-index-1))
         print self.src_blocks_colored[index],
@@ -259,7 +315,12 b' class Demo:'
                               (fname,index,nblocks-index-1))
             print block,
         sys.stdout.flush()
-            
+
+    def runlines(self,source):
+        """Execute a string with one or more lines of code"""
+
+        exec source in self.user_ns
+        
     def __call__(self,index=None):
         """run a block of the demo.
 
@@ -269,12 +330,9 b' class Demo:'
         prints 'Block n/N, and N is the total, so it would be very odd to use
         zero-indexing here."""
 
-        if index is None and self.finished:
-            print 'Demo finished.  Use reset() if you want to rerun it.'
-            return
+        index = self._get_index(index)
         if index is None:
-            index = self.block_index
-        self._validate_index(index)
+            return
         try:
             next_block = self.src_blocks[index]
             self.block_index += 1
@@ -294,7 +352,7 b' class Demo:'
             try:
                 save_argv = sys.argv
                 sys.argv = self.sys_argv
-                exec next_block in self.user_ns
+                self.runlines(next_block)
             finally:
                 sys.argv = save_argv
             
@@ -309,3 +367,52 b' class Demo:'
             print marquee('Use reset() if you want to rerun it.')
             self.finished = True
 
+class IPythonDemo(Demo):
+    """Class for interactive demos with IPython's input processing applied.
+
+    This subclasses Demo, but instead of executing each block by the Python
+    interpreter (via exec), it actually calls IPython on it, so that any input
+    filters which may be in place are applied to the input block.
+
+    If you have an interactive environment which exposes special input
+    processing, you can use this class instead to write demo scripts which
+    operate exactly as if you had typed them interactively.  The default Demo
+    class requires the input to be valid, pure Python code.
+    """
+
+    def runlines(self,source):
+        """Execute a string with one or more lines of code"""
+
+        self.runlines(source)
+        
+class LineDemo(Demo):
+    """Demo where each line is executed as a separate block.
+
+    The input script should be valid Python code.
+
+    This class doesn't require any markup at all, and it's meant for simple
+    scripts (with no nesting or any kind of indentation) which consist of
+    multiple lines of input to be executed, one at a time, as if they had been
+    typed in the interactive prompt."""
+    
+    def reload(self):
+        """Reload source from disk and initialize state."""
+        # read data and parse into blocks
+        src_b           = [l for l in file_readlines(self.fname) if l.strip()]
+        nblocks         = len(src_b)
+        self.src        = os.linesep.join(file_readlines(self.fname))
+        self._silent    = [False]*nblocks
+        self._auto      = [True]*nblocks
+        self.auto_all   = True
+        self.nblocks    = nblocks
+        self.src_blocks = src_b
+
+        # also build syntax-highlighted source
+        self.src_blocks_colored = map(self.ip_colorize,self.src_blocks)
+
+        # ensure clean namespace and seek offset
+        self.reset()
+
+class IPythonLineDemo(IPythonDemo,LineDemo):
+    """Variant of the LineDemo class whose input is processed by IPython."""
+    pass

IPython/genutils.py

@@ -5,7 +5,7 b' General purpose utilities.'
 This is a grab-bag of stuff I find useful in most programs I write. Some of
 these things are also convenient when working at the command line.
 
-$Id: genutils.py 1110 2006-01-30 20:43:30Z vivainio $"""
+$Id: genutils.py 1126 2006-02-06 02:31:40Z fperez $"""
 
 #*****************************************************************************
 #       Copyright (C) 2001-2006 Fernando Perez. <[e-mail]>
@@ -529,11 +529,18 b' def filefind(fname,alt_dirs = None):'
 #----------------------------------------------------------------------------
 def file_read(filename):
     """Read a file and close it.  Returns the file source."""
-    fobj=open(filename,'r');
+    fobj = open(filename,'r');
     source = fobj.read();
     fobj.close()
     return source
 
+def file_readlines(filename):
+    """Read a file and close it.  Returns the file source using readlines()."""
+    fobj = open(filename,'r');
+    lines = fobj.readlines();
+    fobj.close()
+    return lines
+
 #----------------------------------------------------------------------------
 def target_outdated(target,deps):
     """Determine whether a target is out of date.

doc/ChangeLog

@@ -1,3 +1,20 b''
[phone]  Fernando Perez  <[e-mail]>
+
+	* IPython/demo.py (IPythonDemo): Add new classes to the demo
+	facilities, for demos processed by the IPython input filter
+	(IPythonDemo), and for running a script one-line-at-a-time as a
+	demo, both for pure Python (LineDemo) and for IPython-processed
+	input (IPythonLineDemo).  After a request by Dave Kohel, from the
+	SAGE team.
+	(Demo.edit): added and edit() method to the demo objects, to edit
+	the in-memory copy of the last executed block.
+
+	* IPython/Magic.py (magic_edit): add '-r' option for 'raw'
+	processing to %edit, %macro and %save.  These commands can now be
+	invoked on the unprocessed input as it was typed by the user
+	(without any prefilters applied).  After requests by the SAGE team
+	at SAGE days 2006: http://modular.ucsd.edu/sage/days1/schedule.html.
+
 2006-02-01  Ville Vainio  <[e-mail]>  
 
 	* setup.py, eggsetup.py: easy_install ipython==dev works