wrap helpstring output to 80 cols

IPython/config/application.py

@@ -1,362 +1,379 b''
 # encoding: utf-8
 """
 A base class for a configurable application.
 
 Authors:
 
 * Brian Granger
 * Min RK
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from copy import deepcopy
 import logging
 import re
 import sys
 
 from IPython.config.configurable import SingletonConfigurable
 from IPython.config.loader import (
     KeyValueConfigLoader, PyFileConfigLoader, Config, ArgumentError
 )
 
 from IPython.utils.traitlets import (
     Unicode, List, Int, Enum, Dict, Instance
 )
 from IPython.utils.importstring import import_item
-from IPython.utils.text import indent
+from IPython.utils.text import indent, wrap_paragraphs, dedent
+
+#-----------------------------------------------------------------------------
+# function for re-wrapping a helpstring
+#-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Descriptions for the various sections
 #-----------------------------------------------------------------------------
 
 flag_description = """
 Flags are command-line arguments passed as '--<flag>'.
 These take no parameters, unlike regular key-value arguments.
 They are typically used for setting boolean flags, or enabling
 modes that involve setting multiple options together.
+
+Flags *always* begin with '--', never just one '-'.
 """.strip() # trim newlines of front and back
 
 alias_description = """
 These are commonly set parameters, given abbreviated aliases for convenience.
 They are set in the same `name=value` way as class parameters, where
 <name> is replaced by the real parameter for which it is an alias.
 """.strip() # trim newlines of front and back
 
 keyvalue_description = """
 Parameters are set from command-line arguments of the form:
 `Class.trait=value`.  Parameters will *never* be prefixed with '-'.
 This line is evaluated in Python, so simple expressions are allowed, e.g.
     `C.a='range(3)'`   For setting C.a=[0,1,2]
 """.strip() # trim newlines of front and back
 
 #-----------------------------------------------------------------------------
 # Application class
 #-----------------------------------------------------------------------------
 
 
 class ApplicationError(Exception):
     pass
 
 
 class Application(SingletonConfigurable):
     """A singleton application with full configuration support."""
 
     # The name of the application, will usually match the name of the command
     # line application
     name = Unicode(u'application')
 
     # The description of the application that is printed at the beginning
     # of the help.
     description = Unicode(u'This is an application.')
     # default section descriptions
     flag_description = Unicode(flag_description)
     alias_description = Unicode(alias_description)
     keyvalue_description = Unicode(keyvalue_description)
     
 
     # A sequence of Configurable subclasses whose config=True attributes will
     # be exposed at the command line.
     classes = List([])
 
     # The version string of this application.
     version = Unicode(u'0.0')
 
     # The log level for the application
     log_level = Enum((0,10,20,30,40,50), default_value=logging.WARN,
                      config=True,
                      help="Set the log level.")
     
     # the alias map for configurables
     aliases = Dict(dict(log_level='Application.log_level'))
     
     # flags for loading Configurables or store_const style flags
     # flags are loaded from this dict by '--key' flags
     # this must be a dict of two-tuples, the first element being the Config/dict
     # and the second being the help string for the flag
     flags = Dict()
+    def _flags_changed(self, name, old, new):
+        """ensure flags dict is valid"""
+        for key,value in new.iteritems():
+            assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
+            assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
+            assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
+        
     
     # subcommands for launching other applications
     # if this is not empty, this will be a parent Application
-    # this must be a dict of two-tuples, the first element being the application class/import string
+    # this must be a dict of two-tuples, 
+    # the first element being the application class/import string
     # and the second being the help string for the subcommand
     subcommands = Dict()
     # parse_command_line will initialize a subapp, if requested
     subapp = Instance('IPython.config.application.Application', allow_none=True)
     
     # extra command-line arguments that don't set config values
     extra_args = List(Unicode)
     
 
     def __init__(self, **kwargs):
         SingletonConfigurable.__init__(self, **kwargs)
         # Add my class to self.classes so my attributes appear in command line
         # options.
         self.classes.insert(0, self.__class__)
         
-        # ensure self.flags dict is valid
-        for key,value in self.flags.iteritems():
-            assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
-            assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
-            assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)
         self.init_logging()
 
     def _config_changed(self, name, old, new):
         SingletonConfigurable._config_changed(self, name, old, new)
         self.log.debug('Config changed:')
         self.log.debug(repr(new))
 
     def init_logging(self):
         """Start logging for this application.
 
         The default is to log to stdout using a StreaHandler. The log level
         starts at loggin.WARN, but this can be adjusted by setting the 
         ``log_level`` attribute.
         """
         self.log = logging.getLogger(self.__class__.__name__)
         self.log.setLevel(self.log_level)
         self._log_handler = logging.StreamHandler()
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
         self.log.addHandler(self._log_handler)
     
     def initialize(self, argv=None):
         """Do the basic steps to configure me.
         
         Override in subclasses.
         """
         self.parse_command_line(argv)
         
     
     def start(self):
         """Start the app mainloop.
         
         Override in subclasses.
         """
         if self.subapp is not None:
             return self.subapp.start()
     
     def _log_level_changed(self, name, old, new):
         """Adjust the log level when log_level is set."""
         self.log.setLevel(new)
     
     def print_alias_help(self):
-        """print the alias part of the help"""
+        """Print the alias part of the help."""
         if not self.aliases:
             return
         
         lines = ['Aliases']
         lines.append('-'*len(lines[0]))
-        lines.append(self.alias_description)
         lines.append('')
+        for p in wrap_paragraphs(self.alias_description):
+            lines.append(p)
+            lines.append('')
         
         classdict = {}
         for cls in self.classes:
             # include all parents (up to, but excluding Configurable) in available names
             for c in cls.mro()[:-3]:
                 classdict[c.__name__] = c
         
         for alias, longname in self.aliases.iteritems():
             classname, traitname = longname.split('.',1)
             cls = classdict[classname]
             
             trait = cls.class_traits(config=True)[traitname]
             help = cls.class_get_trait_help(trait)
             help = help.replace(longname, "%s (%s)"%(alias, longname), 1)
             lines.append(help)
         lines.append('')
         print '\n'.join(lines)
     
     def print_flag_help(self):
-        """print the flag part of the help"""
+        """Print the flag part of the help."""
         if not self.flags:
             return
         
         lines = ['Flags']
         lines.append('-'*len(lines[0]))
-        lines.append(self.flag_description)
         lines.append('')
+        for p in wrap_paragraphs(self.flag_description):
+            lines.append(p)
+            lines.append('')
         
         for m, (cfg,help) in self.flags.iteritems():
             lines.append('--'+m)
-            lines.append(indent(help.strip(), flatten=True))
+            lines.append(indent(dedent(help.strip())))
         lines.append('')
         print '\n'.join(lines)
     
     def print_subcommands(self):
-        """print the subcommand part of the help"""
+        """Print the subcommand part of the help."""
         if not self.subcommands:
             return
         
         lines = ["Subcommands"]
         lines.append('-'*len(lines[0]))
         for subc, (cls,help) in self.subcommands.iteritems():
             lines.append("%s : %s"%(subc, cls))
             if help:
-                lines.append(indent(help.strip(), flatten=True))
+                lines.append(indent(dedent(help.strip())))
         lines.append('')
         print '\n'.join(lines)
     
     def print_help(self, classes=False):
         """Print the help for each Configurable class in self.classes.
         
-        If classes=False (the default), only flags and aliases are printed
+        If classes=False (the default), only flags and aliases are printed.
         """
         self.print_subcommands()
         self.print_flag_help()
         self.print_alias_help()
         
         if classes:
             if self.classes:
                 print "Class parameters"
                 print "----------------"
-                print self.keyvalue_description
                 print
+                for p in wrap_paragraphs(self.keyvalue_description):
+                    print p
+                    print
         
             for cls in self.classes:
                 cls.class_print_help()
                 print
         else:
             print "To see all available configurables, use `--help-all`"
             print
 
     def print_description(self):
         """Print the application description."""
-        print self.description
-        print
+        for p in wrap_paragraphs(self.description):
+            print p
+            print
 
     def print_version(self):
         """Print the version string."""
         print self.version
 
     def update_config(self, config):
         """Fire the traits events when the config is updated."""
         # Save a copy of the current config.
         newconfig = deepcopy(self.config)
         # Merge the new config into the current one.
         newconfig._merge(config)
         # Save the combined config as self.config, which triggers the traits
         # events.
         self.config = newconfig
     
     def initialize_subcommand(self, subc, argv=None):
-        """Initialize a subcommand with argv"""
+        """Initialize a subcommand with argv."""
         subapp,help = self.subcommands.get(subc)
         
         if isinstance(subapp, basestring):
             subapp = import_item(subapp)
         
         # clear existing instances
         self.__class__.clear_instance()
         # instantiate
         self.subapp = subapp.instance()
         # and initialize subapp
         self.subapp.initialize(argv)
         
     def parse_command_line(self, argv=None):
         """Parse the command line arguments."""
         argv = sys.argv[1:] if argv is None else argv
 
         if self.subcommands and len(argv) > 0:
             # we have subcommands, and one may have been specified
             subc, subargv = argv[0], argv[1:]
             if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
                 # it's a subcommand, and *not* a flag or class parameter
                 return self.initialize_subcommand(subc, subargv)
             
         if '-h' in argv or '--help' in argv or '--help-all' in argv:
             self.print_description()
             self.print_help('--help-all' in argv)
             self.exit(0)
 
         if '--version' in argv:
             self.print_version()
             self.exit(0)
         
         loader = KeyValueConfigLoader(argv=argv, aliases=self.aliases,
                                         flags=self.flags)
         try:
             config = loader.load_config()
         except ArgumentError as e:
             self.log.fatal(str(e))
             self.print_description()
             self.print_help()
             self.exit(1)
         self.update_config(config)
         # store unparsed args in extra_args
         self.extra_args = loader.extra_args
 
     def load_config_file(self, filename, path=None):
         """Load a .py based config file by filename and path."""
         loader = PyFileConfigLoader(filename, path=path)
         config = loader.load_config()
         self.update_config(config)
 
     def exit(self, exit_status=0):
         self.log.debug("Exiting application: %s" % self.name)
         sys.exit(exit_status)
 
 #-----------------------------------------------------------------------------
 # utility functions, for convenience
 #-----------------------------------------------------------------------------
 
 def boolean_flag(name, configurable, set_help='', unset_help=''):
-    """helper for building basic --trait, --no-trait flags
+    """Helper for building basic --trait, --no-trait flags.
     
     Parameters
     ----------
     
     name : str
         The name of the flag.
     configurable : str
         The 'Class.trait' string of the trait to be set/unset with the flag
     set_help : unicode
         help string for --name flag
     unset_help : unicode
         help string for --no-name flag
     
     Returns
     -------
     
     cfg : dict
         A dict with two keys: 'name', and 'no-name', for setting and unsetting
         the trait, respectively.
     """
     # default helpstrings
     set_help = set_help or "set %s=True"%configurable
     unset_help = unset_help or "set %s=False"%configurable
     
     cls,trait = configurable.split('.')
     
     setter = {cls : {trait : True}}
     unsetter = {cls : {trait : False}}
     return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
+

IPython/config/configurable.py

@@ -1,279 +1,283 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 A base class for objects that are configurable.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 * Min RK
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
-from copy import deepcopy
 import datetime
+from copy import deepcopy
 
 from loader import Config
 from IPython.utils.traitlets import HasTraits, Instance
-from IPython.utils.text import indent
+from IPython.utils.text import indent, wrap_paragraphs
 
 
 #-----------------------------------------------------------------------------
 # Helper classes for Configurables
 #-----------------------------------------------------------------------------
 
 
 class ConfigurableError(Exception):
     pass
 
 
 class MultipleInstanceError(ConfigurableError):
     pass
 
 #-----------------------------------------------------------------------------
 # Configurable implementation
 #-----------------------------------------------------------------------------
 
 class Configurable(HasTraits):
 
     config = Instance(Config,(),{})
     created = None
 
     def __init__(self, **kwargs):
-        """Create a conigurable given a config config.
+        """Create a configurable given a config config.
 
         Parameters
         ----------
         config : Config
             If this is empty, default values are used. If config is a 
             :class:`Config` instance, it will be used to configure the
             instance.
         
         Notes
         -----
         Subclasses of Configurable must call the :meth:`__init__` method of
         :class:`Configurable` *before* doing anything else and using 
         :func:`super`::
         
             class MyConfigurable(Configurable):
                 def __init__(self, config=None):
                     super(MyConfigurable, self).__init__(config)
                     # Then any other code you need to finish initialization.
 
         This ensures that instances will be configured properly.
         """
         config = kwargs.pop('config', None)
         if config is not None:
             # We used to deepcopy, but for now we are trying to just save
             # by reference.  This *could* have side effects as all components
             # will share config. In fact, I did find such a side effect in
             # _config_changed below. If a config attribute value was a mutable type
             # all instances of a component were getting the same copy, effectively
             # making that a class attribute.
             # self.config = deepcopy(config)
             self.config = config
         # This should go second so individual keyword arguments override 
         # the values in config.
         super(Configurable, self).__init__(**kwargs)
         self.created = datetime.datetime.now()
 
     #-------------------------------------------------------------------------
     # Static trait notifiations
     #-------------------------------------------------------------------------
 
     def _config_changed(self, name, old, new):
         """Update all the class traits having ``config=True`` as metadata.
 
         For any class trait with a ``config`` metadata attribute that is
         ``True``, we update the trait with the value of the corresponding
         config entry.
         """
         # Get all traits with a config metadata entry that is True
         traits = self.traits(config=True)
 
         # We auto-load config section for this class as well as any parent
         # classes that are Configurable subclasses.  This starts with Configurable
         # and works down the mro loading the config for each section.
         section_names = [cls.__name__ for cls in \
             reversed(self.__class__.__mro__) if 
             issubclass(cls, Configurable) and issubclass(self.__class__, cls)]
 
         for sname in section_names:
             # Don't do a blind getattr as that would cause the config to 
             # dynamically create the section with name self.__class__.__name__.
             if new._has_section(sname):
                 my_config = new[sname]
                 for k, v in traits.iteritems():
                     # Don't allow traitlets with config=True to start with
                     # uppercase.  Otherwise, they are confused with Config
                     # subsections.  But, developers shouldn't have uppercase
                     # attributes anyways! (PEP 6)
                     if k[0].upper()==k[0] and not k.startswith('_'):
                         raise ConfigurableError('Configurable traitlets with '
                         'config=True must start with a lowercase so they are '
                         'not confused with Config subsections: %s.%s' % \
                         (self.__class__.__name__, k))
                     try:
                         # Here we grab the value from the config
                         # If k has the naming convention of a config
                         # section, it will be auto created.
                         config_value = my_config[k]
                     except KeyError:
                         pass
                     else:
                         # print "Setting %s.%s from %s.%s=%r" % \
                         #     (self.__class__.__name__,k,sname,k,config_value)
                         # We have to do a deepcopy here if we don't deepcopy the entire
                         # config object. If we don't, a mutable config_value will be
                         # shared by all instances, effectively making it a class attribute.
                         setattr(self, k, deepcopy(config_value))
 
     @classmethod
     def class_get_help(cls):
         """Get the help string for this class in ReST format."""
         cls_traits = cls.class_traits(config=True)
         final_help = []
         final_help.append(u'%s options' % cls.__name__)
         final_help.append(len(final_help[0])*u'-')
         for k,v in cls.class_traits(config=True).iteritems():
             help = cls.class_get_trait_help(v)
             final_help.append(help)
         return '\n'.join(final_help)
     
     @classmethod
     def class_get_trait_help(cls, trait):
-        """Get the help string for a single """
+        """Get the help string for a single trait."""
         lines = []
         header = "%s.%s : %s" % (cls.__name__, trait.name, trait.__class__.__name__)
+        lines.append(header)
         try:
             dvr = repr(trait.get_default_value())
         except Exception:
             dvr = None # ignore defaults we can't construct
         if dvr is not None:
-            header += ' [default: %s]'%dvr
-        lines.append(header)
+            if len(dvr) > 64:
+                dvr = dvr[:61]+'...'
+            lines.append(indent('Default: %s'%dvr, 4))
+        if 'Enum' in trait.__class__.__name__:
+            # include Enum choices
+            lines.append(indent('Choices: %r'%(trait.values,)))
         
         help = trait.get_metadata('help')
         if help is not None:
-            lines.append(indent(help.strip(), flatten=True))
-        if 'Enum' in trait.__class__.__name__:
-            # include Enum choices
-            lines.append(indent('Choices: %r'%(trait.values,), flatten=True))
+            help = '\n'.join(wrap_paragraphs(help, 76))
+            lines.append(indent(help, 4))
         return '\n'.join(lines)
 
     @classmethod
     def class_print_help(cls):
+        """Get the help string for a single trait and print it."""
         print cls.class_get_help()
 
 
 class SingletonConfigurable(Configurable):
     """A configurable that only allows one instance.
 
     This class is for classes that should only have one instance of itself
     or *any* subclass. To create and retrieve such a class use the
     :meth:`SingletonConfigurable.instance` method.
     """
 
     _instance = None
     
     @classmethod
     def _walk_mro(cls):
         """Walk the cls.mro() for parent classes that are also singletons
         
         For use in instance()
         """
         
         for subclass in cls.mro():
             if issubclass(cls, subclass) and \
                     issubclass(subclass, SingletonConfigurable) and \
                     subclass != SingletonConfigurable:
                 yield subclass
     
     @classmethod
     def clear_instance(cls):
         """unset _instance for this class and singleton parents.
         """
         if not cls.initialized():
             return
         for subclass in cls._walk_mro():
             if isinstance(subclass._instance, cls):
                 # only clear instances that are instances
                 # of the calling class
                 subclass._instance = None
             
     @classmethod
     def instance(cls, *args, **kwargs):
         """Returns a global instance of this class.
 
         This method create a new instance if none have previously been created
         and returns a previously created instance is one already exists.
 
         The arguments and keyword arguments passed to this method are passed
         on to the :meth:`__init__` method of the class upon instantiation.
 
         Examples
         --------
 
         Create a singleton class using instance, and retrieve it::
 
             >>> from IPython.config.configurable import SingletonConfigurable
             >>> class Foo(SingletonConfigurable): pass
             >>> foo = Foo.instance()
             >>> foo == Foo.instance()
             True
 
         Create a subclass that is retrived using the base class instance::
 
             >>> class Bar(SingletonConfigurable): pass
             >>> class Bam(Bar): pass
             >>> bam = Bam.instance()
             >>> bam == Bar.instance()
             True
         """
         # Create and save the instance
         if cls._instance is None:
             inst = cls(*args, **kwargs)
             # Now make sure that the instance will also be returned by
             # parent classes' _instance attribute.
             for subclass in cls._walk_mro():
                 subclass._instance = inst
         
         if isinstance(cls._instance, cls):
             return cls._instance
         else:
             raise MultipleInstanceError(
                 'Multiple incompatible subclass instances of '
                 '%s are being created.' % cls.__name__
             )
 
     @classmethod
     def initialized(cls):
         """Has an instance been created?"""
         return hasattr(cls, "_instance") and cls._instance is not None
 
 
 class LoggingConfigurable(Configurable):
     """A parent class for Configurables that log.
     
     Subclasses have a log trait, and the default behavior
     is to get the logger from the currently running Application
     via Application.instance().log.
     """
     
     log = Instance('logging.Logger')
     def _log_default(self):
         from IPython.config.application import Application
         return Application.instance().log
         
         
  No newline at end of file

IPython/utils/text.py

@@ -1,560 +1,613 b''
 # encoding: utf-8
 """
 Utilities for working with strings and text.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __main__
 
 import os
 import re
 import shutil
+import textwrap
 from string import Formatter
 
 from IPython.external.path import path
 
 from IPython.utils.io import nlprint
 from IPython.utils.data import flatten
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 
 def unquote_ends(istr):
     """Remove a single pair of quotes from the endpoints of a string."""
 
     if not istr:
         return istr
     if (istr[0]=="'" and istr[-1]=="'") or \
        (istr[0]=='"' and istr[-1]=='"'):
         return istr[1:-1]
     else:
         return istr
 
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
 # FIXME: We need to reimplement type specific displayhook and then add this
 # back as a custom printer. This should also be moved outside utils into the
 # core.
 
 # def print_lsstring(arg):
 #     """ Prettier (non-repr-like) and more informative printer for LSString """
 #     print "LSString (.p, .n, .l, .s available). Value:"
 #     print arg
 # 
 # 
 # print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
         .l (or .list) : value as list (the list itself).
         .n (or .nlstr): value as a string, joined on newlines.
         .s (or .spstr): value as a string, joined on spaces.
         .p (or .paths): list of path objects
 
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [path(p) for p in self if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
     def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable)
 
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
 
         If field is specified, the match must occur in the specified
         whitespace-separated field.
 
         Examples::
 
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
             a.grep('chm', field=-1)
         """
 
         def match_target(s):
             if field is None:
                 return s
             parts = s.split()
             try:
                 tgt = parts[field]
                 return tgt
             except IndexError:
                 return ""
 
         if isinstance(pattern, basestring):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
             return SList([el for el in self if pred(match_target(el))])
         else:
             return SList([el for el in self if not pred(match_target(el))])
 
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
 
         Allows quick awk-like usage of string lists.
 
         Example data (in var a, created by 'a = !ls -l')::
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython
 
         a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
         a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
         (note the joining by space).
         a.fields(-1) is ['ChangeLog', 'IPython']
 
         IndexErrors are ignored.
 
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
 
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
 
         return res
 
     def sort(self,field= None,  nums = False):
         """ sort by specified fields (see fields())
 
         Example::
             a.sort(1, nums = True)
 
         Sorts a by second field, in numerical order (so that 21 > 3)
 
         """
 
         #decorate, sort, undecorate
         if field is not None:
             dsu = [[SList([line]).fields(field),  line] for line in self]
         else:
             dsu = [[line,  line] for line in self]
         if nums:
             for i in range(len(dsu)):
                 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
                 try:
                     n = int(numstr)
                 except ValueError:
                     n = 0;
                 dsu[i][0] = n
 
 
         dsu.sort()
         return SList([t[1] for t in dsu])
 
 
 # FIXME: We need to reimplement type specific displayhook and then add this
 # back as a custom printer. This should also be moved outside utils into the
 # core.
 
 # def print_slist(arg):
 #     """ Prettier (non-repr-like) and more informative printer for SList """
 #     print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
 #     if hasattr(arg,  'hideonce') and arg.hideonce:
 #         arg.hideonce = False
 #         return
 # 
 #     nlprint(arg)
 # 
 # print_slist = result_display.when_type(SList)(print_slist)
 
 
 def esc_quotes(strng):
     """Return the input string with single and double quotes escaped out"""
 
     return strng.replace('"','\\"').replace("'","\\'")
 
 
 def make_quoted_expr(s):
     """Return string s in appropriate quotes, using raw string if possible.
 
     XXX - example removed because it caused encoding errors in documentation
     generation.  We need a new example that doesn't contain invalid chars.
 
     Note the use of raw string and padding at the end to allow trailing
     backslash.
     """
 
     tail = ''
     tailpadding = ''
     raw  = ''
     ucode = 'u'
     if "\\" in s:
         raw = 'r'
         if s.endswith('\\'):
             tail = '[:-1]'
             tailpadding = '_'
     if '"' not in s:
         quote = '"'
     elif "'" not in s:
         quote = "'"
     elif '"""' not in s and not s.endswith('"'):
         quote = '"""'
     elif "'''" not in s and not s.endswith("'"):
         quote = "'''"
     else:
         # give up, backslash-escaped string will do
         return '"%s"' % esc_quotes(s)
     res = ucode + raw + quote + s + tailpadding + quote + tail
     return res
 
 
 def qw(words,flat=0,sep=None,maxsplit=-1):
     """Similar to Perl's qw() operator, but with some more options.
 
     qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)
 
     words can also be a list itself, and with flat=1, the output will be
     recursively flattened.
 
     Examples:
 
     >>> qw('1 2')
     ['1', '2']
 
     >>> qw(['a b','1 2',['m n','p q']])
     [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]
 
     >>> qw(['a b','1 2',['m n','p q']],flat=1)
     ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
     """
 
     if isinstance(words, basestring):
         return [word.strip() for word in words.split(sep,maxsplit)
                 if word and not word.isspace() ]
     if flat:
         return flatten(map(qw,words,[1]*len(words)))
     return map(qw,words)
 
 
 def qwflat(words,sep=None,maxsplit=-1):
     """Calls qw(words) in flat mode. It's just a convenient shorthand."""
     return qw(words,1,sep,maxsplit)
 
 
 def qw_lol(indata):
     """qw_lol('a b') -> [['a','b']],
     otherwise it's just a call to qw().
 
     We need this to make sure the modules_some keys *always* end up as a
     list of lists."""
 
     if isinstance(indata, basestring):
         return [qw(indata)]
     else:
         return qw(indata)
 
 
 def grep(pat,list,case=1):
     """Simple minded grep-like function.
     grep(pat,list) returns occurrences of pat in list, None on failure.
 
     It only does simple string matching, with no support for regexps. Use the
     option case=0 for case-insensitive matching."""
 
     # This is pretty crude. At least it should implement copying only references
     # to the original data in case it's big. Now it copies the data for output.
     out=[]
     if case:
         for term in list:
             if term.find(pat)>-1: out.append(term)
     else:
         lpat=pat.lower()
         for term in list:
             if term.lower().find(lpat)>-1: out.append(term)
 
     if len(out): return out
     else: return None
 
 
 def dgrep(pat,*opts):
     """Return grep() on dir()+dir(__builtins__).
 
     A very common use of grep() when working interactively."""
 
     return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)
 
 
 def idgrep(pat):
     """Case-insensitive dgrep()"""
 
     return dgrep(pat,0)
 
 
 def igrep(pat,list):
     """Synonym for case-insensitive grep."""
 
     return grep(pat,list,case=0)
 
 
 def indent(instr,nspaces=4, ntabs=0, flatten=False):
     """Indent a string a given number of spaces or tabstops.
 
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
     
     Parameters
     ----------
     
     instr : basestring
         The string to be indented.
     nspaces : int (default: 4)
         The number of spaces to be indented.
     ntabs : int (default: 0)
         The number of tabs to be indented.
     flatten : bool (default: False)
         Whether to scrub existing indentation.  If True, all lines will be
         aligned to the same indentation.  If False, existing indentation will
         be strictly increased.
     
     Returns
     -------
     
     str|unicode : string indented by ntabs and nspaces.
     
     """
     if instr is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     if flatten:
         pat = re.compile(r'^\s*', re.MULTILINE)
     else:
         pat = re.compile(r'^', re.MULTILINE)
     outstr = re.sub(pat, ind, instr)
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
 
 def native_line_ends(filename,backup=1):
     """Convert (in-place) a file to line-ends native to the current OS.
 
     If the optional backup argument is given as false, no backup of the
     original file is left.  """
 
     backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
 
     bak_filename = filename + backup_suffixes[os.name]
 
     original = open(filename).read()
     shutil.copy2(filename,bak_filename)
     try:
         new = open(filename,'wb')
         new.write(os.linesep.join(original.splitlines()))
         new.write(os.linesep) # ALWAYS put an eol at the end of the file
         new.close()
     except:
         os.rename(bak_filename,filename)
     if not backup:
         try:
             os.remove(bak_filename)
         except:
             pass
 
 
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input.
 
     :Examples:
 
         In [7]: list_strings('A single string')
         Out[7]: ['A single string']
 
         In [8]: list_strings(['A single string in a list'])
         Out[8]: ['A single string in a list']
 
         In [9]: list_strings(['A','list','of','strings'])
         Out[9]: ['A', 'list', 'of', 'strings']
     """
 
     if isinstance(arg,basestring): return [arg]
     else: return arg
 
 
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'.
 
     :Examples:
 
         In [16]: marquee('A test',40)
         Out[16]: '**************** A test ****************'
 
         In [17]: marquee('A test',40,'-')
         Out[17]: '---------------- A test ----------------'
 
         In [18]: marquee('A test',40,' ')
         Out[18]: '                 A test                 '
 
     """
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)/len(mark)/2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 
 ini_spaces_re = re.compile(r'^(\s+)')
 
 def num_ini_spaces(strng):
     """Return the number of initial spaces in a string"""
 
     ini_spaces = ini_spaces_re.match(strng)
     if ini_spaces:
         return ini_spaces.end()
     else:
         return 0
 
 
 def format_screen(strng):
     """Format a string for screen printing.
 
     This removes some latex-type format codes."""
     # Paragraph continue
     par_re = re.compile(r'\\$',re.MULTILINE)
     strng = par_re.sub('',strng)
     return strng
 
+def dedent(text):
+    """Equivalent of textwrap.dedent that ignores unindented first line.
+    
+    This means it will still dedent strings like:
+    '''foo
+    is a bar
+    '''
+    
+    For use in wrap_paragraphs.
+    """
+    
+    if text.startswith('\n'):
+        # text starts with blank line, don't ignore the first line
+        return textwrap.dedent(text)
+    
+    # split first line
+    splits = text.split('\n',1)
+    if len(splits) == 1:
+        # only one line
+        return textwrap.dedent(text)
+    
+    first, rest = splits
+    # dedent everything but the first line
+    rest = textwrap.dedent(rest)
+    return '\n'.join([first, rest])
+
+def wrap_paragraphs(text, ncols=80):
+    """Wrap multiple paragraphs to fit a specified width.
+    
+    This is equivalent to textwrap.wrap, but with support for multiple
+    paragraphs, as separated by empty lines.
+    
+    Returns
+    -------
+    
+    list of complete paragraphs, wrapped to fill `ncols` columns.
+    """
+    paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
+    text = dedent(text).strip()
+    paragraphs = paragraph_re.split(text)[::2] # every other entry is space
+    out_ps = []
+    indent_re = re.compile(r'\n\s+', re.MULTILINE)
+    for p in paragraphs:
+        # presume indentation that survives dedent is meaningful formatting, 
+        # so don't fill unless text is flush.
+        if indent_re.search(p) is None:
+            # wrap paragraph
+            p = textwrap.fill(p, ncols)
+        out_ps.append(p)
+    return out_ps
+    
+
 
 class EvalFormatter(Formatter):
     """A String Formatter that allows evaluation of simple expressions.
     
     Any time a format key is not found in the kwargs,
     it will be tried as an expression in the kwargs namespace.
     
     This is to be used in templating cases, such as the parallel batch
     script templates, where simple arithmetic on arguments is useful.
     
     Examples
     --------
     
     In [1]: f = EvalFormatter()
     In [2]: f.format('{n/4}', n=8)
     Out[2]: '2'
     
     In [3]: f.format('{range(3)}')
     Out[3]: '[0, 1, 2]'
 
     In [4]: f.format('{3*2}')
     Out[4]: '6'
     """
     
     def get_value(self, key, args, kwargs):
         if isinstance(key, (int, long)):
             return args[key]
         elif key in kwargs:
             return kwargs[key]
         else:
             # evaluate the expression using kwargs as namespace
             try:
                 return eval(key, kwargs)
             except Exception:
                 # classify all bad expressions as key errors
                 raise KeyError(key)