Tweak code with suggestions from yesterday.

IPython/config/loader.py

@@ -1,375 +1,376 b''
 # -*- coding: utf-8 -*-
 # coding: utf-8
 """A simple configuration system.
 
 Authors
 -------
 * Brian Granger
 * Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import __builtin__
 import os
 import sys
 
 from IPython.external import argparse
 from IPython.utils.path import filefind
 
 #-----------------------------------------------------------------------------
 # Exceptions
 #-----------------------------------------------------------------------------
 
 
 class ConfigError(Exception):
     pass
 
 
 class ConfigLoaderError(ConfigError):
     pass
 
 #-----------------------------------------------------------------------------
 # Argparse fix
 #-----------------------------------------------------------------------------
 
 # Unfortunately argparse by default prints help messages to stderr instead of
 # stdout.  This makes it annoying to capture long help screens at the command
 # line, since one must know how to pipe stderr, which many users don't know how
 # to do.  So we override the print_help method with one that defaults to
 # stdout and use our class instead.
 
 class ArgumentParser(argparse.ArgumentParser):
     """Simple argparse subclass that prints help to stdout by default."""
     
     def print_help(self, file=None):
         if file is None:
             file = sys.stdout
         return super(ArgumentParser, self).print_help(file)
     
     print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
     
 #-----------------------------------------------------------------------------
 # Config class for holding config information
 #-----------------------------------------------------------------------------
 
 
 class Config(dict):
     """An attribute based dict that can do smart merges."""
 
     def __init__(self, *args, **kwds):
         dict.__init__(self, *args, **kwds)
         # This sets self.__dict__ = self, but it has to be done this way
         # because we are also overriding __setattr__.
         dict.__setattr__(self, '__dict__', self)
 
     def _merge(self, other):
         to_update = {}
         for k, v in other.iteritems():
             if not self.has_key(k):
                 to_update[k] = v
             else: # I have this key
                 if isinstance(v, Config):
                     # Recursively merge common sub Configs
                     self[k]._merge(v)
                 else:
                     # Plain updates for non-Configs
                     to_update[k] = v
 
         self.update(to_update)
 
     def _is_section_key(self, key):
         if key[0].upper()==key[0] and not key.startswith('_'):
             return True
         else:
             return False
 
     def __contains__(self, key):
         if self._is_section_key(key):
             return True
         else:
             return super(Config, self).__contains__(key)
     # .has_key is deprecated for dictionaries.
     has_key = __contains__
 
     def _has_section(self, key):
         if self._is_section_key(key):
             if super(Config, self).__contains__(key):
                 return True
         return False
 
     def copy(self):
         return type(self)(dict.copy(self))
 
     def __copy__(self):
         return self.copy()
 
     def __deepcopy__(self, memo):
         import copy
         return type(self)(copy.deepcopy(self.items()))
 
     def __getitem__(self, key):
         # Because we use this for an exec namespace, we need to delegate
         # the lookup of names in __builtin__ to itself.  This means
         # that you can't have section or attribute names that are 
         # builtins.
         try:
             return getattr(__builtin__, key)
         except AttributeError:
             pass
         if self._is_section_key(key):
             try:
                 return dict.__getitem__(self, key)
             except KeyError:
                 c = Config()
                 dict.__setitem__(self, key, c)
                 return c
         else:
             return dict.__getitem__(self, key)
 
     def __setitem__(self, key, value):
         # Don't allow names in __builtin__ to be modified.
         if hasattr(__builtin__, key):
             raise ConfigError('Config variable names cannot have the same name '
                               'as a Python builtin: %s' % key)
         if self._is_section_key(key):
             if not isinstance(value, Config):
                 raise ValueError('values whose keys begin with an uppercase '
                                  'char must be Config instances: %r, %r' % (key, value))
         else:
             dict.__setitem__(self, key, value)
 
     def __getattr__(self, key):
         try:
             return self.__getitem__(key)
         except KeyError, e:
             raise AttributeError(e)
 
     def __setattr__(self, key, value):
         try:
             self.__setitem__(key, value)
         except KeyError, e:
             raise AttributeError(e)
 
     def __delattr__(self, key):
         try:
             dict.__delitem__(self, key)
         except KeyError, e:
             raise AttributeError(e)
 
 
 #-----------------------------------------------------------------------------
 # Config loading classes
 #-----------------------------------------------------------------------------
 
 
 class ConfigLoader(object):
     """A object for loading configurations from just about anywhere.
     
     The resulting configuration is packaged as a :class:`Struct`.
     
     Notes
     -----
     A :class:`ConfigLoader` does one thing: load a config from a source 
     (file, command line arguments) and returns the data as a :class:`Struct`.
     There are lots of things that :class:`ConfigLoader` does not do.  It does
     not implement complex logic for finding config files.  It does not handle
     default values or merge multiple configs.  These things need to be 
     handled elsewhere.
     """
 
     def __init__(self):
         """A base class for config loaders.
         
         Examples
         --------
         
         >>> cl = ConfigLoader()
         >>> config = cl.load_config()
         >>> config
         {}
         """
         self.clear()
 
     def clear(self):
         self.config = Config()
 
     def load_config(self):
         """Load a config from somewhere, return a :class:`Config` instance.
         
         Usually, this will cause self.config to be set and then returned.
         However, in most cases, :meth:`ConfigLoader.clear` should be called
         to erase any previous state.
         """
         self.clear()
         return self.config
 
 
 class FileConfigLoader(ConfigLoader):
     """A base class for file based configurations.
 
     As we add more file based config loaders, the common logic should go
     here.
     """
     pass
 
 
 class PyFileConfigLoader(FileConfigLoader):
     """A config loader for pure python files.
     
     This calls execfile on a plain python file and looks for attributes
     that are all caps.  These attribute are added to the config Struct.
     """
 
     def __init__(self, filename, path=None):
         """Build a config loader for a filename and path.
 
         Parameters
         ----------
         filename : str
             The file name of the config file.
         path : str, list, tuple
             The path to search for the config file on, or a sequence of
             paths to try in order.
         """
         super(PyFileConfigLoader, self).__init__()
         self.filename = filename
         self.path = path
         self.full_filename = ''
         self.data = None
 
     def load_config(self):
         """Load the config from a file and return it as a Struct."""
         self.clear()
         self._find_file()
         self._read_file_as_dict()
         self._convert_to_config()
         return self.config
 
     def _find_file(self):
         """Try to find the file by searching the paths."""
         self.full_filename = filefind(self.filename, self.path)
 
     def _read_file_as_dict(self):
         """Load the config file into self.config, with recursive loading."""
         # This closure is made available in the namespace that is used
         # to exec the config file.  This allows users to call
         # load_subconfig('myconfig.py') to load config files recursively.
         # It needs to be a closure because it has references to self.path
         # and self.config.  The sub-config is loaded with the same path
         # as the parent, but it uses an empty config which is then merged
         # with the parents.
         def load_subconfig(fname):
             loader = PyFileConfigLoader(fname, self.path)
             try:
                 sub_config = loader.load_config()
             except IOError:
                 # Pass silently if the sub config is not there. This happens
                 # when a user us using a profile, but not the default config.
                 pass
             else:
                 self.config._merge(sub_config)
 
         # Again, this needs to be a closure and should be used in config
         # files to get the config being loaded.
         def get_config():
             return self.config
 
         namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
-        conf_filename = self.full_filename.encode(sys.getfilesystemencoding())
+        fs_encoding = sys.getfilesystemencoding() or 'ascii'
+        conf_filename = self.full_filename.encode(fs_encoding)
         execfile(conf_filename, namespace)
 
     def _convert_to_config(self):
         if self.data is None:
             ConfigLoaderError('self.data does not exist')
 
 
 class CommandLineConfigLoader(ConfigLoader):
     """A config loader for command line arguments.
 
     As we add more command line based loaders, the common logic should go
     here.
     """
 
 
 class ArgParseConfigLoader(CommandLineConfigLoader):
 
     def __init__(self, argv=None, *parser_args, **parser_kw):
         """Create a config loader for use with argparse.
 
         Parameters
         ----------
 
         argv : optional, list
           If given, used to read command-line arguments from, otherwise
           sys.argv[1:] is used.
 
         parser_args : tuple
           A tuple of positional arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
 
         parser_kw : dict
           A tuple of keyword arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
         """
         super(CommandLineConfigLoader, self).__init__()
         if argv == None:
             argv = sys.argv[1:]
         self.argv = argv
         self.parser_args = parser_args
         self.version = parser_kw.pop("version", None)
         kwargs = dict(argument_default=argparse.SUPPRESS)
         kwargs.update(parser_kw)
         self.parser_kw = kwargs
 
     def load_config(self, args=None):
         """Parse command line arguments and return as a Struct.
 
         Parameters
         ----------
 
         args : optional, list
           If given, a list with the structure of sys.argv[1:] to parse
           arguments from. If not given, the instance's self.argv attribute
           (given at construction time) is used."""
         self.clear()
         if args is None:
             args = self.argv
         self._create_parser()
         self._parse_args(args)
         self._convert_to_config()
         return self.config
 
     def get_extra_args(self):
         if hasattr(self, 'extra_args'):
             return self.extra_args
         else:
             return []
 
     def _create_parser(self):
         self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
         self._add_arguments()
 
     def _add_arguments(self):
         raise NotImplementedError("subclasses must implement _add_arguments")
 
     def _parse_args(self, args):
         """self.parser->self.parsed_data""" 
         self.parsed_data, self.extra_args = self.parser.parse_known_args(args)
 
     def _convert_to_config(self):
         """self.parsed_data->self.config"""
         for k, v in vars(self.parsed_data).iteritems():
             exec_str = 'self.config.' + k + '= v'
             exec exec_str in locals(), globals()
 
 

IPython/core/application.py

@@ -1,461 +1,465 b''
 # encoding: utf-8
 """
 An application for IPython.
 
 All top-level applications should use the classes in this module for
 handling configuration and creating componenets.
 
 The job of an :class:`Application` is to create the master configuration 
 object and then create the configurable objects, passing the config to them.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 
 Notes
 -----
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import logging
 import os
 import sys
 
 from IPython.core import release, crashhandler
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
 from IPython.config.loader import (
     PyFileConfigLoader,
     ArgParseConfigLoader,
     Config,
 )
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 class ApplicationError(Exception):
     pass
 
 
 class BaseAppConfigLoader(ArgParseConfigLoader):
     """Default command line options for IPython based applications."""
 
     def _add_ipython_dir(self, parser):
         """Add the --ipython-dir option to the parser."""
         paa = parser.add_argument
         paa('--ipython-dir', 
             dest='Global.ipython_dir',type=unicode,
             help=
             """Set to override default location of the IPython directory
             IPYTHON_DIR, stored as Global.ipython_dir.  This can also be 
             specified through the environment variable IPYTHON_DIR.""",
             metavar='Global.ipython_dir')
 
     def _add_log_level(self, parser):
         """Add the --log-level option to the parser."""
         paa = parser.add_argument
         paa('--log-level',
             dest="Global.log_level",type=int,
             help='Set the log level (0,10,20,30,40,50).  Default is 30.',
             metavar='Global.log_level')
             
     def _add_version(self, parser):
         """Add the --version option to the parser."""
         parser.add_argument('--version', action="version",
                                                 version=self.version)
 
     def _add_arguments(self):
         self._add_ipython_dir(self.parser)
         self._add_log_level(self.parser)
         self._add_version(self.parser)
 
 
 class Application(object):
     """Load a config, construct configurables and set them running.
 
     The configuration of an application can be done via three different Config
     objects, which are loaded and ultimately merged into a single one used
     from that point on by the app. These are:
 
     1. default_config: internal defaults, implemented in code.
     2. file_config: read from the filesystem.
     3. command_line_config: read from the system's command line flags.
 
     During initialization, 3 is actually read before 2, since at the
     command-line one may override the location of the file to be read.  But the
     above is the order in which the merge is made.
     """
 
     name = u'ipython'
     description = 'IPython: an enhanced interactive Python shell.'
     #: Usage message printed by argparse. If None, auto-generate
     usage = None
     #: The command line config loader.  Subclass of ArgParseConfigLoader.
     command_line_loader = BaseAppConfigLoader
     #: The name of the config file to load, determined at runtime
     config_file_name = None
     #: The name of the default config file. Track separately from the actual
     #: name because some logic happens only if we aren't using the default.
     default_config_file_name = u'ipython_config.py'
     default_log_level = logging.WARN
     #: Set by --profile option
     profile_name = None
     #: User's ipython directory, typically ~/.ipython or ~/.config/ipython/
     ipython_dir = None
     #: Internal defaults, implemented in code.
     default_config = None
     #: Read from the filesystem.
     file_config = None
     #: Read from the system's command line flags.
     command_line_config = None
     #: The final config that will be passed to the main object.
     master_config = None
     #: A reference to the argv to be used (typically ends up being sys.argv[1:])
     argv = None
     #: extra arguments computed by the command-line loader
     extra_args = None
     #: The class to use as the crash handler.
     crash_handler_class = crashhandler.CrashHandler
 
     # Private attributes
     _exiting = False
     _initialized = False
 
     def __init__(self, argv=None):
         self.argv = sys.argv[1:] if argv is None else argv
         self.init_logger()
 
     def init_logger(self):
         self.log = logging.getLogger(self.__class__.__name__)
         # This is used as the default until the command line arguments are read.
         self.log.setLevel(self.default_log_level)
         self._log_handler = logging.StreamHandler()
         self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
         self._log_handler.setFormatter(self._log_formatter)
         self.log.addHandler(self._log_handler)
 
     def _set_log_level(self, level):
         self.log.setLevel(level)
 
     def _get_log_level(self):
         return self.log.level
 
     log_level = property(_get_log_level, _set_log_level)
 
     def initialize(self):
         """Initialize the application.
 
         Loads all configuration information and sets all application state, but
         does not start any relevant processing (typically some kind of event
         loop).
 
         Once this method has been called, the application is flagged as
         initialized and the method becomes a no-op."""
         
         if self._initialized:
             return
 
         # The first part is protected with an 'attempt' wrapper, that will log
         # failures with the basic system traceback machinery.  Once our crash
         # handler is in place, we can let any subsequent exception propagate,
         # as our handler will log it with much better detail than the default.
         self.attempt(self.create_crash_handler)
 
         # Configuration phase
         # Default config (internally hardwired in application code)
         self.create_default_config()
         self.log_default_config()
         self.set_default_config_log_level()
 
         # Command-line config
         self.pre_load_command_line_config()
         self.load_command_line_config()
         self.set_command_line_config_log_level()
         self.post_load_command_line_config()
         self.log_command_line_config()
 
         # Find resources needed for filesystem access, using information from
         # the above two
         self.find_ipython_dir()
         self.find_resources()
         self.find_config_file_name()
         self.find_config_file_paths()
 
         # File-based config
         self.pre_load_file_config()
         self.load_file_config()
         self.set_file_config_log_level()
         self.post_load_file_config()
         self.log_file_config()
 
         # Merge all config objects into a single one the app can then use
         self.merge_configs()
         self.log_master_config()
 
         # Construction phase
         self.pre_construct()
         self.construct()
         self.post_construct()
 
         # Done, flag as such and
         self._initialized = True
 
     def start(self):
         """Start the application."""
         self.initialize()
         self.start_app()
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
     def create_crash_handler(self):
         """Create a crash handler, typically setting sys.excepthook to it."""
         self.crash_handler = self.crash_handler_class(self)
         sys.excepthook = self.crash_handler
 
     def create_default_config(self):
         """Create defaults that can't be set elsewhere.
 
         For the most part, we try to set default in the class attributes
         of Configurables.  But, defaults the top-level Application (which is
         not a HasTraits or Configurables) are not set in this way.  Instead
         we set them here.  The Global section is for variables like this that
         don't belong to a particular configurable.
         """
         c = Config()
         c.Global.ipython_dir = get_ipython_dir()
         c.Global.log_level = self.log_level
         self.default_config = c
 
     def log_default_config(self):
         self.log.debug('Default config loaded:')
         self.log.debug(repr(self.default_config))
 
     def set_default_config_log_level(self):
         try:
             self.log_level = self.default_config.Global.log_level
         except AttributeError:
             # Fallback to the default_log_level class attribute
             pass
 
     def create_command_line_config(self):
         """Create and return a command line config loader."""
         return self.command_line_loader(
             self.argv,
             description=self.description, 
             version=release.version,
             usage=self.usage
         )
 
     def pre_load_command_line_config(self):
         """Do actions just before loading the command line config."""
         pass
 
     def load_command_line_config(self):
         """Load the command line config."""
         loader = self.create_command_line_config()
         self.command_line_config = loader.load_config()
         self.extra_args = loader.get_extra_args()
 
     def set_command_line_config_log_level(self):
         try:
             self.log_level = self.command_line_config.Global.log_level
         except AttributeError:
             pass
 
     def post_load_command_line_config(self):
         """Do actions just after loading the command line config."""
         pass
 
     def log_command_line_config(self):
         self.log.debug("Command line config loaded:")
         self.log.debug(repr(self.command_line_config))
 
     def find_ipython_dir(self):
         """Set the IPython directory.
 
         This sets ``self.ipython_dir``, but the actual value that is passed to
         the application is kept in either ``self.default_config`` or
         ``self.command_line_config``.  This also adds ``self.ipython_dir`` to
         ``sys.path`` so config files there can be referenced by other config
         files.
         """
 
         try:
             self.ipython_dir = self.command_line_config.Global.ipython_dir
         except AttributeError:
             self.ipython_dir = self.default_config.Global.ipython_dir
         sys.path.append(os.path.abspath(self.ipython_dir))
         if not os.path.isdir(self.ipython_dir):
             os.makedirs(self.ipython_dir, mode=0777)
         self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir)
 
     def find_resources(self):
         """Find other resources that need to be in place.
 
         Things like cluster directories need to be in place to find the
         config file.  These happen right after the IPython directory has
         been set.
         """
         pass
 
     def find_config_file_name(self):
         """Find the config file name for this application.
 
         This must set ``self.config_file_name`` to the filename of the
         config file to use (just the filename). The search paths for the
         config file are set in :meth:`find_config_file_paths` and then passed
         to the config file loader where they are resolved to an absolute path.
 
         If a profile has been set at the command line, this will resolve it.
         """
         try:
             self.config_file_name = self.command_line_config.Global.config_file
         except AttributeError:
             pass
         else:
             return
 
         try:
             self.profile_name = self.command_line_config.Global.profile
         except AttributeError:
             # Just use the default as there is no profile
             self.config_file_name = self.default_config_file_name
         else:
             # Use the default config file name and profile name if set
             # to determine the used config file name.
             name_parts = self.default_config_file_name.split('.')
             name_parts.insert(1, u'_' + self.profile_name + u'.')
             self.config_file_name = ''.join(name_parts)
 
     def find_config_file_paths(self):
         """Set the search paths for resolving the config file.
 
         This must set ``self.config_file_paths`` to a sequence of search
         paths to pass to the config file loader.
         """
         # Include our own profiles directory last, so that users can still find
         # our shipped copies of builtin profiles even if they don't have them
         # in their local ipython directory.
         prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
         self.config_file_paths = (os.getcwdu(), self.ipython_dir, prof_dir)
 
     def pre_load_file_config(self):
         """Do actions before the config file is loaded."""
         pass
 
     def load_file_config(self, suppress_errors=True):
         """Load the config file.
         
         This tries to load the config file from disk.  If successful, the
         ``CONFIG_FILE`` config variable is set to the resolved config file
         location.  If not successful, an empty config is used.
+        
+        By default, errors in loading config are handled, and a warning
+        printed on screen. For testing, the suppress_errors option is set
+        to False, so errors will make tests fail.
         """
         self.log.debug("Attempting to load config file: %s" %
                        self.config_file_name)
         loader = PyFileConfigLoader(self.config_file_name,
                                     path=self.config_file_paths)
         try:
             self.file_config = loader.load_config()
             self.file_config.Global.config_file = loader.full_filename
         except IOError:
             # Only warn if the default config file was NOT being used.
             if not self.config_file_name==self.default_config_file_name:
                 self.log.warn("Config file not found, skipping: %s" %
                                self.config_file_name, exc_info=True)
             self.file_config = Config()
         except:
             if not suppress_errors:     # For testing purposes
                 raise
             self.log.warn("Error loading config file: %s" %
                           self.config_file_name, exc_info=True)
             self.file_config = Config()
 
     def set_file_config_log_level(self):
         # We need to keeep self.log_level updated.  But we only use the value
         # of the file_config if a value was not specified at the command
         # line, because the command line overrides everything.
         if not hasattr(self.command_line_config.Global, 'log_level'):
             try:
                 self.log_level = self.file_config.Global.log_level
             except AttributeError:
                 pass # Use existing value
 
     def post_load_file_config(self):
         """Do actions after the config file is loaded."""
         pass
 
     def log_file_config(self):
         if hasattr(self.file_config.Global, 'config_file'):
             self.log.debug("Config file loaded: %s" %
                            self.file_config.Global.config_file)
             self.log.debug(repr(self.file_config))
 
     def merge_configs(self):
         """Merge the default, command line and file config objects."""
         config = Config()
         config._merge(self.default_config)
         config._merge(self.file_config)
         config._merge(self.command_line_config)
 
         # XXX fperez - propose to Brian we rename master_config to simply
         # config, I think this is going to be heavily used in examples and
         # application code and the name is shorter/easier to find/remember.
         # For now, just alias it...
         self.master_config = config
         self.config = config
 
     def log_master_config(self):
         self.log.debug("Master config created:")
         self.log.debug(repr(self.master_config))
 
     def pre_construct(self):
         """Do actions after the config has been built, but before construct."""
         pass
 
     def construct(self):
         """Construct the main objects that make up this app."""
         self.log.debug("Constructing main objects for application")
 
     def post_construct(self):
         """Do actions after construct, but before starting the app."""
         pass
 
     def start_app(self):
         """Actually start the app."""
         self.log.debug("Starting application")
 
     #-------------------------------------------------------------------------
     # Utility methods
     #-------------------------------------------------------------------------
 
     def exit(self, exit_status=0):
         if self._exiting:
             pass
         else:
             self.log.debug("Exiting application: %s" % self.name)
             self._exiting = True
             sys.exit(exit_status)
 
     def attempt(self, func):
         try:
             func()
         except SystemExit:
             raise
         except:
             self.log.critical("Aborting application: %s" % self.name,
                               exc_info=True)
             self.exit(0)
 

IPython/core/tests/test_compilerop.py

@@ -1,71 +1,74 b''
 # coding: utf-8
 """Tests for the compilerop module.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010 The IPython Development Team.
 #
 #  Distributed under the terms of the BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib imports
 import linecache
+import sys
 
 # Third-party imports
 import nose.tools as nt
 
 # Our own imports
 from IPython.core import compilerop
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 
 def test_code_name():
     code = 'x=1'
     name = compilerop.code_name(code)
     nt.assert_true(name.startswith('<ipython-input-0'))
 
 
 def test_code_name2():
     code = 'x=1'
     name = compilerop.code_name(code, 9)
     nt.assert_true(name.startswith('<ipython-input-9'))
 
 
 def test_compiler():
     """Test the compiler correctly compiles and caches inputs
     """
     cp = compilerop.CachingCompiler()
     ncache = len(linecache.cache)
     cp('x=1', 'single')
     nt.assert_true(len(linecache.cache) > ncache)
 
-def test_compiler_unicode():
-    import sys
+def setUp():
+    # Check we're in a proper Python 2 environment (some imports, such
+    # as GTK, can change the default encoding, which can hide bugs.)
     nt.assert_equal(sys.getdefaultencoding(), "ascii")
-    
+
+def test_compiler_unicode():
     cp = compilerop.CachingCompiler()
     ncache = len(linecache.cache)
     cp(u"t = 'žćčšđ'", "single")
     nt.assert_true(len(linecache.cache) > ncache)
 
 def test_compiler_check_cache():
     """Test the compiler properly manages the cache.
     """
     # Rather simple-minded tests that just exercise the API
     cp = compilerop.CachingCompiler()
     cp('x=1', 'single', 99)
     # Ensure now that after clearing the cache, our entries survive
     cp.check_cache()
     for k in linecache.cache:
         if k.startswith('<ipython-input-99'):
             break
     else:
         raise AssertionError('Entry for input-99 missing from linecache')

IPython/core/tests/test_history.py

@@ -1,117 +1,119 b''
 # coding: utf-8
 """Tests for the IPython tab-completion machinery.
 """
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 # stdlib
 import os
 import sys
 import unittest
 
 # third party
 import nose.tools as nt
 
 # our own packages
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.core.history import HistoryManager, extract_hist_ranges
 
-def test_history():
+def setUp():
     nt.assert_equal(sys.getdefaultencoding(), "ascii")
+
+def test_history():
     ip = get_ipython()
     with TemporaryDirectory() as tmpdir:
         #tmpdir = '/software/temp'
         histfile = os.path.realpath(os.path.join(tmpdir, 'history.sqlite'))
         # Ensure that we restore the history management that we mess with in
         # this test doesn't affect the IPython instance used by the test suite
         # beyond this test.
         hist_manager_ori = ip.history_manager
         try:
             ip.history_manager = HistoryManager(shell=ip)
             ip.history_manager.hist_file = histfile
             ip.history_manager.init_db()  # Has to be called after changing file
             ip.history_manager.reset()
             print 'test',histfile
             hist = ['a=1', 'def f():\n    test = 1\n    return test', u"b='€Æ¾÷ß'"]
             for i, h in enumerate(hist, start=1):
                 ip.history_manager.store_inputs(i, h)
             
             ip.history_manager.db_log_output = True
             # Doesn't match the input, but we'll just check it's stored.
             ip.history_manager.output_hist_reprs[3].append("spam")
             ip.history_manager.store_output(3)
             
             nt.assert_equal(ip.history_manager.input_hist_raw, [''] + hist)
             
             # Check lines were written to DB
             c = ip.history_manager.db.execute("SELECT source_raw FROM history")
             nt.assert_equal([x for x, in c], hist)
               
             # New session
             ip.history_manager.reset()
             newcmds = ["z=5","class X(object):\n    pass", "k='p'"]
             for i, cmd in enumerate(newcmds, start=1):
                 ip.history_manager.store_inputs(i, cmd)
             gothist = ip.history_manager.get_range(start=1, stop=4)
             nt.assert_equal(list(gothist), zip([0,0,0],[1,2,3], newcmds))
             # Previous session:
             gothist = ip.history_manager.get_range(-1, 1, 4)
             nt.assert_equal(list(gothist), zip([1,1,1],[1,2,3], hist))
             
             # Check get_hist_tail
             gothist = ip.history_manager.get_tail(4, output=True,
                                                     include_latest=True)
             expected = [(1, 3, (hist[-1], ["spam"])),
                         (2, 1, (newcmds[0], None)),
                         (2, 2, (newcmds[1], None)),
                         (2, 3, (newcmds[2], None)),]
             nt.assert_equal(list(gothist), expected)
             
             gothist = ip.history_manager.get_tail(2)
             expected = [(2, 1, newcmds[0]),
                         (2, 2, newcmds[1])]
             nt.assert_equal(list(gothist), expected)
             
             # Check get_hist_search
             gothist = ip.history_manager.search("*test*")
             nt.assert_equal(list(gothist), [(1,2,hist[1])] )
             gothist = ip.history_manager.search("b*", output=True)
             nt.assert_equal(list(gothist), [(1,3,(hist[2],["spam"]))] )
             
             # Cross testing: check that magic %save can get previous session.
             testfilename = os.path.realpath(os.path.join(tmpdir, "test.py"))
             ip.magic_save(testfilename + " ~1/1-3")
             testfile = open(testfilename, "r")
             nt.assert_equal(testfile.read().decode("utf-8"),
                     "# coding: utf-8\n" + "\n".join(hist))
             
             # Duplicate line numbers - check that it doesn't crash, and
             # gets a new session
             ip.history_manager.store_inputs(1, "rogue")
             nt.assert_equal(ip.history_manager.session_number, 3)
         finally:
             # Restore history manager
             ip.history_manager = hist_manager_ori
 
 
 def test_extract_hist_ranges():
     instr = "1 2/3 ~4/5-6 ~4/7-~4/9 ~9/2-~7/5"
     expected = [(0, 1, 2),  # 0 == current session
                 (2, 3, 4),
                 (-4, 5, 7),
                 (-4, 7, 10),
                 (-9, 2, None),  # None == to end
                 (-8, 1, None),
                 (-7, 1, 6)]
     actual = list(extract_hist_ranges(instr))
     nt.assert_equal(actual, expected)
     
 def test_magic_rerun():
     """Simple test for %rerun (no args -> rerun last line)"""
     ip = get_ipython()
     ip.run_cell("a = 10")
     ip.run_cell("a += 1")
     nt.assert_equal(ip.user_ns["a"], 11)
     ip.run_cell("%rerun")
     nt.assert_equal(ip.user_ns["a"], 12)

IPython/testing/iptest.py

@@ -1,437 +1,441 b''
 # -*- coding: utf-8 -*-
 """IPython Test Suite Runner.
 
 This module provides a main entry point to a user script to test IPython
 itself from the command line. There are two ways of running this script:
 
 1. With the syntax `iptest all`.  This runs our entire test suite by
    calling this script (with different arguments) or trial recursively.  This
    causes modules and package to be tested in different processes, using nose
    or trial where appropriate.
 2. With the regular nose syntax, like `iptest -vvs IPython`.  In this form
    the script simply calls nose, but with special command line flags and
    plugins loaded.
 
 For now, this script requires that both nose and twisted are installed.  This
 will change in the future.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2009  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import os
 import os.path as path
 import signal
 import sys
 import subprocess
 import tempfile
 import time
 import warnings
 
 # Note: monkeypatch!
 # We need to monkeypatch a small problem in nose itself first, before importing
 # it for actual use.  This should get into nose upstream, but its release cycle
 # is slow and we need it for our parametric tests to work correctly.
 from IPython.testing import nosepatch
 # Now, proceed to import nose itself
 import nose.plugins.builtin
 from nose.core import TestProgram
 
 # Our own imports
 from IPython.utils.path import get_ipython_module_path
 from IPython.utils.process import find_cmd, pycmd2argv
 from IPython.utils.sysinfo import sys_info
 
 from IPython.testing import globalipapp
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
 
 pjoin = path.join
 
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 
 #-----------------------------------------------------------------------------
 # Warnings control
 #-----------------------------------------------------------------------------
 
 # Twisted generates annoying warnings with Python 2.6, as will do other code
 # that imports 'sets' as of today
 warnings.filterwarnings('ignore', 'the sets module is deprecated',
                         DeprecationWarning )
 
 # This one also comes from Twisted
 warnings.filterwarnings('ignore', 'the sha module is deprecated',
                         DeprecationWarning)
 
 # Wx on Fedora11 spits these out
 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
                         UserWarning)
 
 #-----------------------------------------------------------------------------
 # Logic for skipping doctests
 #-----------------------------------------------------------------------------
 
 def test_for(mod):
     """Test to see if mod is importable."""
     try:
         __import__(mod)
     except (ImportError, RuntimeError):
         # GTK reports Runtime error if it can't be initialized even if  it's
         # importable.
         return False
     else:
         return True
 
 # Global dict where we can store information on what we have and what we don't
 # have available at test run time
 have = {}
 
 have['curses'] = test_for('_curses')
 have['wx'] = test_for('wx')
 have['wx.aui'] = test_for('wx.aui')
 have['zope.interface'] = test_for('zope.interface')
 have['twisted'] = test_for('twisted')
 have['foolscap'] = test_for('foolscap')
 have['pexpect'] = test_for('pexpect')
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 def report():
     """Return a string with a summary report of test-related variables."""
 
     out = [ sys_info(), '\n']
 
     avail = []
     not_avail = []
     
     for k, is_avail in have.items():
         if is_avail:
             avail.append(k)
         else:
             not_avail.append(k)
 
     if avail:
         out.append('\nTools and libraries available at test time:\n')
         avail.sort()
         out.append('   ' + ' '.join(avail)+'\n')
 
     if not_avail:
         out.append('\nTools and libraries NOT available at test time:\n')
         not_avail.sort()
         out.append('   ' + ' '.join(not_avail)+'\n')
         
     return ''.join(out)
 
 
 def make_exclude():
     """Make patterns of modules and packages to exclude from testing.
 
     For the IPythonDoctest plugin, we need to exclude certain patterns that
     cause testing problems.  We should strive to minimize the number of
     skipped modules, since this means untested code.
 
     These modules and packages will NOT get scanned by nose at all for tests.
     """
     # Simple utility to make IPython paths more readably, we need a lot of
     # these below
     ipjoin = lambda *paths: pjoin('IPython', *paths)
     
     exclusions = [ipjoin('external'),
                   pjoin('IPython_doctest_plugin'),
                   ipjoin('quarantine'),
                   ipjoin('deathrow'),
                   ipjoin('testing', 'attic'),
                   # This guy is probably attic material
                   ipjoin('testing', 'mkdoctests'),
                   # Testing inputhook will need a lot of thought, to figure out
                   # how to have tests that don't lock up with the gui event
                   # loops in the picture
                   ipjoin('lib', 'inputhook'),
                   # Config files aren't really importable stand-alone
                   ipjoin('config', 'default'),
                   ipjoin('config', 'profile'),
                   ]
 
     if not have['wx']:
         exclusions.append(ipjoin('lib', 'inputhookwx'))
+        
+    # We do this unconditionally, so that the test suite doesn't import
+    # gtk, changing the default encoding and masking some unicode bugs.
+    exclusions.append(ipjoin('lib', 'inputhookgtk'))
 
     # These have to be skipped on win32 because the use echo, rm, cd, etc.
     # See ticket https://bugs.launchpad.net/bugs/366982
     if sys.platform == 'win32':
         exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
         exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
 
     if not have['pexpect']:
         exclusions.extend([ipjoin('scripts', 'irunner'),
                            ipjoin('lib', 'irunner')])
 
     # This is scary.  We still have things in frontend and testing that
     # are being tested by nose that use twisted.  We need to rethink
     # how we are isolating dependencies in testing.
     if not (have['twisted'] and have['zope.interface'] and have['foolscap']):
         exclusions.extend(
             [ipjoin('testing', 'parametric'),
              ipjoin('testing', 'util'),
              ipjoin('testing', 'tests', 'test_decorators_trial'),
              ] )
 
     # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
     if sys.platform == 'win32':
         exclusions = [s.replace('\\','\\\\') for s in exclusions]
 
     return exclusions
 
 
 class IPTester(object):
     """Call that calls iptest or trial in a subprocess.
     """
     #: string, name of test runner that will be called
     runner = None
     #: list, parameters for test runner
     params = None
     #: list, arguments of system call to be made to call test runner
     call_args = None
     #: list, process ids of subprocesses we start (for cleanup)
     pids = None
     
     def __init__(self, runner='iptest', params=None):
         """Create new test runner."""
         p = os.path
         if runner == 'iptest':
             iptest_app = get_ipython_module_path('IPython.testing.iptest')
             self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
         elif runner == 'trial':
             # For trial, it needs to be installed system-wide
             self.runner = pycmd2argv(p.abspath(find_cmd('trial')))
         else:
             raise Exception('Not a valid test runner: %s' % repr(runner))
         if params is None:
             params = []
         if isinstance(params, str):
             params = [params]
         self.params = params
 
         # Assemble call
         self.call_args = self.runner+self.params
 
         # Store pids of anything we start to clean up on deletion, if possible
         # (on posix only, since win32 has no os.kill)
         self.pids = []
 
     if sys.platform == 'win32':
         def _run_cmd(self):
             # On Windows, use os.system instead of subprocess.call, because I
             # was having problems with subprocess and I just don't know enough
             # about win32 to debug this reliably.  Os.system may be the 'old
             # fashioned' way to do it, but it works just fine.  If someone
             # later can clean this up that's fine, as long as the tests run
             # reliably in win32.
             # What types of problems are you having. They may be related to
             # running Python in unboffered mode. BG.
             return os.system(' '.join(self.call_args))
     else:
         def _run_cmd(self):
             # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
             subp = subprocess.Popen(self.call_args)
             self.pids.append(subp.pid)
             # If this fails, the pid will be left in self.pids and cleaned up
             # later, but if the wait call succeeds, then we can clear the
             # stored pid.
             retcode = subp.wait()
             self.pids.pop()
             return retcode
         
     def run(self):
         """Run the stored commands"""
         try:
             return self._run_cmd()
         except:
             import traceback
             traceback.print_exc()
             return 1  # signal failure
 
     def __del__(self):
         """Cleanup on exit by killing any leftover processes."""
 
         if not hasattr(os, 'kill'):
             return
         
         for pid in self.pids:
             try:
                 print 'Cleaning stale PID:', pid
                 os.kill(pid, signal.SIGKILL)
             except OSError:
                 # This is just a best effort, if we fail or the process was
                 # really gone, ignore it.
                 pass     
 
 
 def make_runners():
     """Define the top-level packages that need to be tested.
     """
 
     # Packages to be tested via nose, that only depend on the stdlib
     nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
                      'scripts', 'testing', 'utils' ]
     # The machinery in kernel needs twisted for real testing
     trial_pkg_names = []
 
     # And add twisted ones if conditions are met
     if have['zope.interface'] and have['twisted'] and have['foolscap']:
         # We only list IPython.kernel for testing using twisted.trial as
         # nose and twisted.trial have conflicts that make the testing system
         # unstable.
         trial_pkg_names.append('kernel')
 
     # For debugging this code, only load quick stuff
     #nose_pkg_names = ['core', 'extensions']  # dbg
     #trial_pkg_names = []  # dbg 
 
     # Make fully qualified package names prepending 'IPython.' to our name lists
     nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
     trial_packages = ['IPython.%s' % m for m in trial_pkg_names ]
 
     # Make runners
     runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
     runners.extend([ (v, IPTester('trial', params=v)) for v in trial_packages ])
     
     return runners
 
 
 def run_iptest():
     """Run the IPython test suite using nose.
     
     This function is called when this script is **not** called with the form
     `iptest all`.  It simply calls nose with appropriate command line flags
     and accepts all of the standard nose arguments.
     """
 
     warnings.filterwarnings('ignore', 
         'This will be removed soon.  Use IPython.testing.util instead')
 
     argv = sys.argv + [ '--detailed-errors',  # extra info in tracebacks
                                                 
                         # Loading ipdoctest causes problems with Twisted, but
                         # our test suite runner now separates things and runs
                         # all Twisted tests with trial.
                         '--with-ipdoctest',
                         '--ipdoctest-tests','--ipdoctest-extension=txt',
                         
                         # We add --exe because of setuptools' imbecility (it
                         # blindly does chmod +x on ALL files).  Nose does the
                         # right thing and it tries to avoid executables,
                         # setuptools unfortunately forces our hand here.  This
                         # has been discussed on the distutils list and the
                         # setuptools devs refuse to fix this problem!
                         '--exe',
                         ]
 
     if nose.__version__ >= '0.11':
         # I don't fully understand why we need this one, but depending on what
         # directory the test suite is run from, if we don't give it, 0 tests
         # get run.  Specifically, if the test suite is run from the source dir
         # with an argument (like 'iptest.py IPython.core', 0 tests are run,
         # even if the same call done in this directory works fine).  It appears
         # that if the requested package is in the current dir, nose bails early
         # by default.  Since it's otherwise harmless, leave it in by default
         # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
         argv.append('--traverse-namespace')
 
     # Construct list of plugins, omitting the existing doctest plugin, which
     # ours replaces (and extends).
     plugins = [IPythonDoctest(make_exclude())]
     for p in nose.plugins.builtin.plugins:
         plug = p()
         if plug.name == 'doctest':
             continue
         plugins.append(plug)
 
     # We need a global ipython running in this process
     globalipapp.start_ipython()
     # Now nose can run
     TestProgram(argv=argv, plugins=plugins)
 
 
 def run_iptestall():
     """Run the entire IPython test suite by calling nose and trial.
     
     This function constructs :class:`IPTester` instances for all IPython
     modules and package and then runs each of them.  This causes the modules
     and packages of IPython to be tested each in their own subprocess using
     nose or twisted.trial appropriately.
     """
 
     runners = make_runners()
 
     # Run the test runners in a temporary dir so we can nuke it when finished
     # to clean up any junk files left over by accident.  This also makes it
     # robust against being run in non-writeable directories by mistake, as the
     # temp dir will always be user-writeable.
     curdir = os.getcwd()
     testdir = tempfile.gettempdir()
     os.chdir(testdir)
 
     # Run all test runners, tracking execution time
     failed = []
     t_start = time.time()
     try:
         for (name, runner) in runners:
             print '*'*70
             print 'IPython test group:',name
             res = runner.run()
             if res:
                 failed.append( (name, runner) )
     finally:
         os.chdir(curdir)
     t_end = time.time()
     t_tests = t_end - t_start
     nrunners = len(runners)
     nfail = len(failed)
     # summarize results
     print
     print '*'*70
     print 'Test suite completed for system with the following information:'
     print report()
     print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
     print
     print 'Status:'
     if not failed:
         print 'OK'
     else:
         # If anything went wrong, point out what command to rerun manually to
         # see the actual errors and individual summary
         print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
         for name, failed_runner in failed:
             print '-'*40
             print 'Runner failed:',name
             print 'You may wish to rerun this one individually, with:'
             print ' '.join(failed_runner.call_args)
             print
 
 
 def main():
     for arg in sys.argv[1:]:
         if arg.startswith('IPython'):
             # This is in-process
             run_iptest()
     else:
         # This starts subprocesses
         run_iptestall()
 
 
 if __name__ == '__main__':
     main()