Merge pull request #4318 from minrk/sys.path

fix initial sys.path

IPython/core/application.py

@@ -1,383 +1,383
 # encoding: utf-8
 """
 An application for IPython.
 
 All top-level applications should use the classes in this module for
 handling configuration and creating configurables.
 
 The job of an :class:`Application` is to create the master configuration
 object and then create the configurable objects, passing the config to them.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 * Min RK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import atexit
 import errno
 import glob
 import logging
 import os
 import shutil
 import sys
 
 from IPython.config.application import Application, catch_config_error
 from IPython.config.loader import ConfigFileNotFound
 from IPython.core import release, crashhandler
 from IPython.core.profiledir import ProfileDir, ProfileDirError
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict, Set, Instance
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 
 #-----------------------------------------------------------------------------
 # Base Application Class
 #-----------------------------------------------------------------------------
 
 # aliases and flags
 
 base_aliases = {
     'profile-dir' : 'ProfileDir.location',
     'profile' : 'BaseIPythonApplication.profile',
     'ipython-dir' : 'BaseIPythonApplication.ipython_dir',
     'log-level' : 'Application.log_level',
     'config' : 'BaseIPythonApplication.extra_config_file',
 }
 
 base_flags = dict(
     debug = ({'Application' : {'log_level' : logging.DEBUG}},
             "set log level to logging.DEBUG (maximize logging output)"),
     quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
             "set log level to logging.CRITICAL (minimize logging output)"),
     init = ({'BaseIPythonApplication' : {
                     'copy_config_files' : True,
                     'auto_create' : True}
             }, """Initialize profile with default config files.  This is equivalent
             to running `ipython profile create <profile>` prior to startup.
             """)
 )
 
 
 class BaseIPythonApplication(Application):
 
     name = Unicode(u'ipython')
     description = Unicode(u'IPython: an enhanced interactive Python shell.')
     version = Unicode(release.version)
 
     aliases = Dict(base_aliases)
     flags = Dict(base_flags)
     classes = List([ProfileDir])
 
     # Track whether the config_file has changed,
     # because some logic happens only if we aren't using the default.
     config_file_specified = Set()
 
     config_file_name = Unicode()
     def _config_file_name_default(self):
         return self.name.replace('-','_') + u'_config.py'
     def _config_file_name_changed(self, name, old, new):
         if new != old:
             self.config_file_specified.add(new)
 
     # The directory that contains IPython's builtin profiles.
     builtin_profile_dir = Unicode(
         os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
     )
 
     config_file_paths = List(Unicode)
     def _config_file_paths_default(self):
         return [os.getcwdu()]
 
     extra_config_file = Unicode(config=True,
     help="""Path to an extra config file to load.
     
     If specified, load this config file in addition to any other IPython config.
     """)
     def _extra_config_file_changed(self, name, old, new):
         try:
             self.config_files.remove(old)
         except ValueError:
             pass
         self.config_file_specified.add(new)
         self.config_files.append(new)
 
     profile = Unicode(u'default', config=True,
         help="""The IPython profile to use."""
     )
 
     def _profile_changed(self, name, old, new):
         self.builtin_profile_dir = os.path.join(
                 get_ipython_package_dir(), u'config', u'profile', new
         )
 
     ipython_dir = Unicode(config=True,
         help="""
         The name of the IPython directory. This directory is used for logging
         configuration (through profiles), history storage, etc. The default
         is usually $HOME/.ipython. This options can also be specified through
         the environment variable IPYTHONDIR.
         """
     )
     def _ipython_dir_default(self):
         d = get_ipython_dir()
-        self._ipython_dir_changed('ipython_dir', '', d)
+        self._ipython_dir_changed('ipython_dir', d, d)
         return d
     
     _in_init_profile_dir = False
     profile_dir = Instance(ProfileDir)
     def _profile_dir_default(self):
         # avoid recursion
         if self._in_init_profile_dir:
             return
         # profile_dir requested early, force initialization
         self.init_profile_dir()
         return self.profile_dir
 
     overwrite = Bool(False, config=True,
         help="""Whether to overwrite existing config files when copying""")
     auto_create = Bool(False, config=True,
         help="""Whether to create profile dir if it doesn't exist""")
 
     config_files = List(Unicode)
     def _config_files_default(self):
         return [self.config_file_name]
 
     copy_config_files = Bool(False, config=True,
         help="""Whether to install the default config files into the profile dir.
         If a new profile is being created, and IPython contains config files for that
         profile, then they will be staged into the new directory.  Otherwise,
         default config files will be automatically generated.
         """)
     
     verbose_crash = Bool(False, config=True,
         help="""Create a massive crash report when IPython encounters what may be an
         internal error.  The default is to append a short message to the
         usual traceback""")
 
     # The class to use as the crash handler.
     crash_handler_class = Type(crashhandler.CrashHandler)
 
     @catch_config_error
     def __init__(self, **kwargs):
         super(BaseIPythonApplication, self).__init__(**kwargs)
         # ensure current working directory exists
         try:
             directory = os.getcwdu()
         except:
             # raise exception
             self.log.error("Current working directory doesn't exist.")
             raise
 
     #-------------------------------------------------------------------------
     # Various stages of Application creation
     #-------------------------------------------------------------------------
 
     def init_crash_handler(self):
         """Create a crash handler, typically setting sys.excepthook to it."""
         self.crash_handler = self.crash_handler_class(self)
         sys.excepthook = self.excepthook
         def unset_crashhandler():
             sys.excepthook = sys.__excepthook__
         atexit.register(unset_crashhandler)
     
     def excepthook(self, etype, evalue, tb):
         """this is sys.excepthook after init_crashhandler
         
         set self.verbose_crash=True to use our full crashhandler, instead of
         a regular traceback with a short message (crash_handler_lite)
         """
         
         if self.verbose_crash:
             return self.crash_handler(etype, evalue, tb)
         else:
             return crashhandler.crash_handler_lite(etype, evalue, tb)
     
     def _ipython_dir_changed(self, name, old, new):
         if old in sys.path:
             sys.path.remove(old)
         sys.path.append(os.path.abspath(new))
         if not os.path.isdir(new):
             os.makedirs(new, mode=0o777)
         readme = os.path.join(new, 'README')
         readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README')
         if not os.path.exists(readme) and os.path.exists(readme_src):
             shutil.copy(readme_src, readme)
         for d in ('extensions', 'nbextensions'):
             path = os.path.join(new, d)
             if not os.path.exists(path):
                 try:
                     os.mkdir(path)
                 except OSError as e:
                     if e.errno != errno.EEXIST:
                         self.log.error("couldn't create path %s: %s", path, e)
         self.log.debug("IPYTHONDIR set to: %s" % new)
 
     def load_config_file(self, suppress_errors=True):
         """Load the config file.
 
         By default, errors in loading config are handled, and a warning
         printed on screen. For testing, the suppress_errors option is set
         to False, so errors will make tests fail.
         """
         self.log.debug("Searching path %s for config files", self.config_file_paths)
         base_config = 'ipython_config.py'
         self.log.debug("Attempting to load config file: %s" %
                        base_config)
         try:
             Application.load_config_file(
                 self,
                 base_config,
                 path=self.config_file_paths
             )
         except ConfigFileNotFound:
             # ignore errors loading parent
             self.log.debug("Config file %s not found", base_config)
             pass
         
         for config_file_name in self.config_files:
             if not config_file_name or config_file_name == base_config:
                 continue
             self.log.debug("Attempting to load config file: %s" %
                            self.config_file_name)
             try:
                 Application.load_config_file(
                     self,
                     config_file_name,
                     path=self.config_file_paths
                 )
             except ConfigFileNotFound:
                 # Only warn if the default config file was NOT being used.
                 if config_file_name in self.config_file_specified:
                     msg = self.log.warn
                 else:
                     msg = self.log.debug
                 msg("Config file not found, skipping: %s", config_file_name)
             except:
                 # For testing purposes.
                 if not suppress_errors:
                     raise
                 self.log.warn("Error loading config file: %s" %
                               self.config_file_name, exc_info=True)
 
     def init_profile_dir(self):
         """initialize the profile dir"""
         self._in_init_profile_dir = True
         if self.profile_dir is not None:
             # already ran
             return
         try:
             # location explicitly specified:
             location = self.config.ProfileDir.location
         except AttributeError:
             # location not specified, find by profile name
             try:
                 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
             except ProfileDirError:
                 # not found, maybe create it (always create default profile)
                 if self.auto_create or self.profile == 'default':
                     try:
                         p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
                     except ProfileDirError:
                         self.log.fatal("Could not create profile: %r"%self.profile)
                         self.exit(1)
                     else:
                         self.log.info("Created profile dir: %r"%p.location)
                 else:
                     self.log.fatal("Profile %r not found."%self.profile)
                     self.exit(1)
             else:
                 self.log.info("Using existing profile dir: %r"%p.location)
         else:
             # location is fully specified
             try:
                 p = ProfileDir.find_profile_dir(location, self.config)
             except ProfileDirError:
                 # not found, maybe create it
                 if self.auto_create:
                     try:
                         p = ProfileDir.create_profile_dir(location, self.config)
                     except ProfileDirError:
                         self.log.fatal("Could not create profile directory: %r"%location)
                         self.exit(1)
                     else:
                         self.log.info("Creating new profile dir: %r"%location)
                 else:
                     self.log.fatal("Profile directory %r not found."%location)
                     self.exit(1)
             else:
                 self.log.info("Using existing profile dir: %r"%location)
 
         self.profile_dir = p
         self.config_file_paths.append(p.location)
         self._in_init_profile_dir = False
 
     def init_config_files(self):
         """[optionally] copy default config files into profile dir."""
         # copy config files
         path = self.builtin_profile_dir
         if self.copy_config_files:
             src = self.profile
 
             cfg = self.config_file_name
             if path and os.path.exists(os.path.join(path, cfg)):
                 self.log.warn("Staging %r from %s into %r [overwrite=%s]"%(
                         cfg, src, self.profile_dir.location, self.overwrite)
                 )
                 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
             else:
                 self.stage_default_config_file()
         else:
             # Still stage *bundled* config files, but not generated ones
             # This is necessary for `ipython profile=sympy` to load the profile
             # on the first go
             files = glob.glob(os.path.join(path, '*.py'))
             for fullpath in files:
                 cfg = os.path.basename(fullpath)
                 if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
                     # file was copied
                     self.log.warn("Staging bundled %s from %s into %r"%(
                             cfg, self.profile, self.profile_dir.location)
                     )
 
 
     def stage_default_config_file(self):
         """auto generate default config file, and stage it into the profile."""
         s = self.generate_config_file()
         fname = os.path.join(self.profile_dir.location, self.config_file_name)
         if self.overwrite or not os.path.exists(fname):
             self.log.warn("Generating default config file: %r"%(fname))
             with open(fname, 'w') as f:
                 f.write(s)
 
     @catch_config_error
     def initialize(self, argv=None):
         # don't hook up crash handler before parsing command-line
         self.parse_command_line(argv)
         self.init_crash_handler()
         if self.subapp is not None:
             # stop here if subapp is taking over
             return
         cl_config = self.config
         self.init_profile_dir()
         self.init_config_files()
         self.load_config_file()
         # enforce cl-opts override configfile opts:
         self.update_config(cl_config)
 

IPython/kernel/tests/test_kernel.py

@@ -1,170 +1,186
 """test the IPython Kernel"""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2013  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import sys
 
 import nose.tools as nt
 
 from IPython.testing import decorators as dec, tools as tt
 from IPython.utils import py3compat
+from IPython.utils.path import locate_profile
 
 from .utils import new_kernel, kernel, TIMEOUT, assemble_output, execute, flush_channels
 
 #-------------------------------------------------------------------------------
 # Tests
 #-------------------------------------------------------------------------------
 
 
 def _check_mp_mode(kc, expected=False, stream="stdout"):
     execute(kc=kc, code="import sys")
     flush_channels(kc)
     msg_id, content = execute(kc=kc, code="print (sys.%s._check_mp_mode())" % stream)
     stdout, stderr = assemble_output(kc.iopub_channel)
     nt.assert_equal(eval(stdout.strip()), expected)
 
 
 # printing tests
 
 def test_simple_print():
     """simple print statement in kernel"""
     with kernel() as kc:
         iopub = kc.iopub_channel
         msg_id, content = execute(kc=kc, code="print ('hi')")
         stdout, stderr = assemble_output(iopub)
         nt.assert_equal(stdout, 'hi\n')
         nt.assert_equal(stderr, '')
         _check_mp_mode(kc, expected=False)
 
 
+def test_sys_path():
+    """test that sys.path doesn't get messed up by default"""
+    with kernel() as kc:
+        msg_id, content = execute(kc=kc, code="import sys; print (repr(sys.path[0]))")
+        stdout, stderr = assemble_output(kc.iopub_channel)
+        nt.assert_equal(stdout, "''\n")
+
+def test_sys_path_profile_dir():
+    """test that sys.path doesn't get messed up when `--profile-dir` is specified"""
+    
+    with new_kernel(['--profile-dir', locate_profile('default')]) as kc:
+        msg_id, content = execute(kc=kc, code="import sys; print (repr(sys.path[0]))")
+        stdout, stderr = assemble_output(kc.iopub_channel)
+        nt.assert_equal(stdout, "''\n")
+
 @dec.knownfailureif(sys.platform == 'win32', "subprocess prints fail on Windows")
 def test_subprocess_print():
     """printing from forked mp.Process"""
     with new_kernel() as kc:
         iopub = kc.iopub_channel
         
         _check_mp_mode(kc, expected=False)
         flush_channels(kc)
         np = 5
         code = '\n'.join([
             "from __future__ import print_function",
             "import multiprocessing as mp",
             "pool = [mp.Process(target=print, args=('hello', i,)) for i in range(%i)]" % np,
             "for p in pool: p.start()",
             "for p in pool: p.join()"
         ])
         
         expected = '\n'.join([
             "hello %s" % i for i in range(np)
         ]) + '\n'
         
         msg_id, content = execute(kc=kc, code=code)
         stdout, stderr = assemble_output(iopub)
         nt.assert_equal(stdout.count("hello"), np, stdout)
         for n in range(np):
             nt.assert_equal(stdout.count(str(n)), 1, stdout)
         nt.assert_equal(stderr, '')
         _check_mp_mode(kc, expected=False)
         _check_mp_mode(kc, expected=False, stream="stderr")
 
 
 def test_subprocess_noprint():
     """mp.Process without print doesn't trigger iostream mp_mode"""
     with kernel() as kc:
         iopub = kc.iopub_channel
         
         np = 5
         code = '\n'.join([
             "import multiprocessing as mp",
             "pool = [mp.Process(target=range, args=(i,)) for i in range(%i)]" % np,
             "for p in pool: p.start()",
             "for p in pool: p.join()"
         ])
         
         msg_id, content = execute(kc=kc, code=code)
         stdout, stderr = assemble_output(iopub)
         nt.assert_equal(stdout, '')
         nt.assert_equal(stderr, '')
 
         _check_mp_mode(kc, expected=False)
         _check_mp_mode(kc, expected=False, stream="stderr")
 
 
 @dec.knownfailureif(sys.platform == 'win32', "subprocess prints fail on Windows")
 def test_subprocess_error():
     """error in mp.Process doesn't crash"""
     with new_kernel() as kc:
         iopub = kc.iopub_channel
         
         code = '\n'.join([
             "import multiprocessing as mp",
             "p = mp.Process(target=int, args=('hi',))",
             "p.start()",
             "p.join()",
         ])
         
         msg_id, content = execute(kc=kc, code=code)
         stdout, stderr = assemble_output(iopub)
         nt.assert_equal(stdout, '')
         nt.assert_true("ValueError" in stderr, stderr)
 
         _check_mp_mode(kc, expected=False)
         _check_mp_mode(kc, expected=False, stream="stderr")
 
 # raw_input tests
 
 def test_raw_input():
     """test [raw_]input"""
     with kernel() as kc:
         iopub = kc.iopub_channel
         
         input_f = "input" if py3compat.PY3 else "raw_input"
         theprompt = "prompt> "
         code = 'print({input_f}("{theprompt}"))'.format(**locals())
         msg_id = kc.execute(code, allow_stdin=True)
         msg = kc.get_stdin_msg(block=True, timeout=TIMEOUT)
         nt.assert_equal(msg['header']['msg_type'], u'input_request')
         content = msg['content']
         nt.assert_equal(content['prompt'], theprompt)
         text = "some text"
         kc.input(text)
         reply = kc.get_shell_msg(block=True, timeout=TIMEOUT)
         nt.assert_equal(reply['content']['status'], 'ok')
         stdout, stderr = assemble_output(iopub)
         nt.assert_equal(stdout, text + "\n")
 
 
 @dec.skipif(py3compat.PY3)
 def test_eval_input():
     """test input() on Python 2"""
     with kernel() as kc:
         iopub = kc.iopub_channel
         
         input_f = "input" if py3compat.PY3 else "raw_input"
         theprompt = "prompt> "
         code = 'print(input("{theprompt}"))'.format(**locals())
         msg_id = kc.execute(code, allow_stdin=True)
         msg = kc.get_stdin_msg(block=True, timeout=TIMEOUT)
         nt.assert_equal(msg['header']['msg_type'], u'input_request')
         content = msg['content']
         nt.assert_equal(content['prompt'], theprompt)
         kc.input("1+1")
         reply = kc.get_shell_msg(block=True, timeout=TIMEOUT)
         nt.assert_equal(reply['content']['status'], 'ok')
         stdout, stderr = assemble_output(iopub)
         nt.assert_equal(stdout, "2\n")
 
 
 def test_help_output():
     """ipython kernel --help-all works"""
     tt.help_all_output_test('kernel')
 

IPython/kernel/tests/utils.py

@@ -1,167 +1,170
 """utilities for testing IPython kernels"""
 
 #-------------------------------------------------------------------------------
 #  Copyright (C) 2013  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-------------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 import atexit
 
 from contextlib import contextmanager
 from subprocess import PIPE
 from Queue import Empty
 
 import nose.tools as nt
 
 from IPython.kernel import KernelManager
 
 #-------------------------------------------------------------------------------
 # Globals
 #-------------------------------------------------------------------------------
 
 STARTUP_TIMEOUT = 60
 TIMEOUT = 15
 
 KM = None
 KC = None
 
 #-------------------------------------------------------------------------------
 # code
 #-------------------------------------------------------------------------------
 
 
-def start_new_kernel():
+def start_new_kernel(argv=None):
     """start a new kernel, and return its Manager and Client"""
     km = KernelManager()
-    km.start_kernel(stdout=PIPE, stderr=PIPE)
+    kwargs = dict(stdout=PIPE, stderr=PIPE)
+    if argv:
+        kwargs['extra_arguments'] = argv
+    km.start_kernel(**kwargs)
     kc = km.client()
     kc.start_channels()
     
     msg_id = kc.kernel_info()
     kc.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT)
     flush_channels(kc)
     return km, kc
 
 def flush_channels(kc=None):
     """flush any messages waiting on the queue"""
     from .test_message_spec import validate_message
     
     if kc is None:
         kc = KC
     for channel in (kc.shell_channel, kc.iopub_channel):
         while True:
             try:
                 msg = channel.get_msg(block=True, timeout=0.1)
             except Empty:
                 break
             else:
                 validate_message(msg)
 
 
 def execute(code='', kc=None, **kwargs):
     """wrapper for doing common steps for validating an execution request"""
     from .test_message_spec import validate_message
     if kc is None:
         kc = KC
     msg_id = kc.execute(code=code, **kwargs)
     reply = kc.get_shell_msg(timeout=TIMEOUT)
     validate_message(reply, 'execute_reply', msg_id)
     busy = kc.get_iopub_msg(timeout=TIMEOUT)
     validate_message(busy, 'status', msg_id)
     nt.assert_equal(busy['content']['execution_state'], 'busy')
     
     if not kwargs.get('silent'):
         pyin = kc.get_iopub_msg(timeout=TIMEOUT)
         validate_message(pyin, 'pyin', msg_id)
         nt.assert_equal(pyin['content']['code'], code)
     
     return msg_id, reply['content']
 
 def start_global_kernel():
     """start the global kernel (if it isn't running) and return its client"""
     global KM, KC
     if KM is None:
         KM, KC = start_new_kernel()
         atexit.register(stop_global_kernel)
     return KC
 
 @contextmanager
 def kernel():
     """Context manager for the global kernel instance
     
     Should be used for most kernel tests
     
     Returns
     -------
     kernel_client: connected KernelClient instance
     """
     yield start_global_kernel()
 
 def uses_kernel(test_f):
     """Decorator for tests that use the global kernel"""
     def wrapped_test():
         with kernel() as kc:
             test_f(kc)
     wrapped_test.__doc__ = test_f.__doc__
     wrapped_test.__name__ = test_f.__name__
     return wrapped_test
 
 def stop_global_kernel():
     """Stop the global shared kernel instance, if it exists"""
     global KM, KC
     KC.stop_channels()
     KC = None
     if KM is None:
         return
     KM.shutdown_kernel(now=True)
     KM = None
 
 @contextmanager
-def new_kernel():
+def new_kernel(argv=None):
     """Context manager for a new kernel in a subprocess
     
     Should only be used for tests where the kernel must not be re-used.
     
     Returns
     -------
     kernel_client: connected KernelClient instance
     """
-    km, kc = start_new_kernel()
+    km, kc = start_new_kernel(argv)
     try:
         yield kc
     finally:
         kc.stop_channels()
         km.shutdown_kernel(now=True)
 
 
 def assemble_output(iopub):
     """assemble stdout/err from an execution"""
     stdout = ''
     stderr = ''
     while True:
         msg = iopub.get_msg(block=True, timeout=1)
         msg_type = msg['msg_type']
         content = msg['content']
         if msg_type == 'status' and content['execution_state'] == 'idle':
             # idle message signals end of output
             break
         elif msg['msg_type'] == 'stream':
             if content['name'] == 'stdout':
                 stdout += content['data']
             elif content['name'] == 'stderr':
                 stderr += content['data']
             else:
                 raise KeyError("bad stream: %r" % content['name'])
         else:
             # other output, ignored
             pass
     return stdout, stderr