Merge with upstream.

IPython/tests/test_genutils.py

@@ -0,0 +1,32 b''
+# encoding: utf-8
+
+"""Tests for genutils.py"""
+
+__docformat__ = "restructuredtext en"
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from IPython import genutils
+
+
+def test_get_home_dir():
+    """Make sure we can get the home directory."""
+    home_dir = genutils.get_home_dir()
+
+def test_get_ipython_dir():
+    """Make sure we can get the ipython directory."""
+    ipdir = genutils.get_ipython_dir()
+
+def test_get_security_dir():
+    """Make sure we can get the ipython/security directory."""
+    sdir = genutils.get_security_dir()
+    
  No newline at end of file

IPython/config/api.py

@@ -16,7 +16,9 b' __docformat__ = "restructuredtext en"'
 #-------------------------------------------------------------------------------
 
 import os
-from IPython.config.cutils import get_home_dir, get_ipython_dir
+from os.path import join as pjoin
+
+from IPython.genutils import get_home_dir, get_ipython_dir
 from IPython.external.configobj import ConfigObj
 
 # Traitlets config imports
@@ -53,7 +55,7 b' class ConfigObjManager(object):'
 
     def write_default_config_file(self):
         ipdir = get_ipython_dir()
-        fname = ipdir + '/' + self.filename
+        fname = pjoin(ipdir, self.filename)
         if not os.path.isfile(fname):
             print "Writing the configuration file to: " + fname
             self.write_config_obj_to_file(fname)
@@ -87,11 +89,11 b' class ConfigObjManager(object):'
 
         # In ipythondir if it is set
         if ipythondir is not None:
-            trythis = ipythondir + '/' + filename
+            trythis = pjoin(ipythondir, filename)
             if os.path.isfile(trythis):
                 return trythis        
 
-        trythis = get_ipython_dir() + '/' + filename
+        trythis = pjoin(get_ipython_dir(), filename)
         if os.path.isfile(trythis):
             return trythis
 

IPython/config/cutils.py

@@ -22,71 +22,6 b' import sys'
 # Normal code begins
 #---------------------------------------------------------------------------
 
-class HomeDirError(Exception):
-    pass
-
-def get_home_dir():
-    """Return the closest possible equivalent to a 'home' directory.
-
-    We first try $HOME.  Absent that, on NT it's $HOMEDRIVE\$HOMEPATH.
-
-    Currently only Posix and NT are implemented, a HomeDirError exception is
-    raised for all other OSes. """
-
-    isdir = os.path.isdir
-    env = os.environ
-    try:
-        homedir = env['HOME']
-        if not isdir(homedir):
-            # in case a user stuck some string which does NOT resolve to a
-            # valid path, it's as good as if we hadn't foud it
-            raise KeyError
-        return homedir
-    except KeyError:
-        if os.name == 'posix':
-            raise HomeDirError,'undefined $HOME, IPython can not proceed.'
-        elif os.name == 'nt':
-            # For some strange reason, win9x returns 'nt' for os.name.
-            try:
-                homedir = os.path.join(env['HOMEDRIVE'],env['HOMEPATH'])
-                if not isdir(homedir):
-                    homedir = os.path.join(env['USERPROFILE'])
-                    if not isdir(homedir):
-                        raise HomeDirError
-                return homedir
-            except:
-                try:
-                    # Use the registry to get the 'My Documents' folder.
-                    import _winreg as wreg
-                    key = wreg.OpenKey(wreg.HKEY_CURRENT_USER,
-                                       "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
-                    homedir = wreg.QueryValueEx(key,'Personal')[0]
-                    key.Close()
-                    if not isdir(homedir):
-                        e = ('Invalid "Personal" folder registry key '
-                             'typically "My Documents".\n'
-                             'Value: %s\n'
-                             'This is not a valid directory on your system.' %
-                             homedir)
-                        raise HomeDirError(e)
-                    return homedir
-                except HomeDirError:
-                    raise
-                except:
-                    return 'C:\\'
-        elif os.name == 'dos':
-            # Desperate, may do absurd things in classic MacOS. May work under DOS.
-            return 'C:\\'
-        else:
-            raise HomeDirError,'support for your operating system not implemented.'
-
-def get_ipython_dir():
-    ipdir_def = '.ipython'
-    home_dir = get_home_dir()
-    ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
-                                           os.path.join(home_dir,ipdir_def)))
-    return ipdir
-
 def import_item(key):
     """
     Import and return bar given the string foo.bar.

IPython/frontend/tests/test_frontendbase.py

@@ -25,6 +25,8 b' except ImportError:'
     import nose
     raise nose.SkipTest("This test requires zope.interface, Twisted and Foolscap")
 
+from IPython.testing.decorators import skip
+
 class FrontEndCallbackChecker(AsyncFrontEndBase):
     """FrontEndBase subclass for checking callbacks"""
     def __init__(self, engine=None, history=None):
@@ -56,12 +58,10 b' class TestAsyncFrontendBase(unittest.TestCase):'
         
         self.fb = FrontEndCallbackChecker(engine=EngineService())
     
-    
     def test_implements_IFrontEnd(self):
         assert(frontendbase.IFrontEnd.implementedBy(
                                     AsyncFrontEndBase))
     
-    
     def test_is_complete_returns_False_for_incomplete_block(self):
         """"""
         
@@ -80,7 +80,6 b' class TestAsyncFrontendBase(unittest.TestCase):'
         
         assert(self.fb.is_complete(block))
     
-    
     def test_blockID_added_to_result(self):
         block = """3+3"""
         
@@ -113,12 +112,11 b' class TestAsyncFrontendBase(unittest.TestCase):'
         d = self.fb.execute("10+10")
         d.addCallback(self.checkCallbacks)
     
-    
     def checkCallbacks(self, result):
         assert(self.fb.updateCalled)
         assert(self.fb.renderResultCalled)
     
-    
+    @skip("This test fails and lead to an unhandled error in a Deferred.")
     def test_error_callback_added_to_execute(self):
         """test that render_error called on execution error"""
         

IPython/frontend/tests/test_process.py

@@ -19,6 +19,9 b' import sys'
 from IPython.frontend._process import PipedProcess
 from IPython.testing import decorators as testdec
 
+
+# FIXME
[e-mail]("This doesn't work under Windows")
 def test_capture_out():
     """ A simple test to see if we can execute a process and get the output.
     """
@@ -29,6 +32,7 b' def test_capture_out():'
     result = s.getvalue().rstrip()
     assert result == '1'
 
+
 # FIXME
 @testdec.skip("This doesn't work under Windows")
 def test_io():
@@ -47,6 +51,8 b' def test_io():'
     assert result == test_string
 
 
+# FIXME
[e-mail]("This doesn't work under Windows")
 def test_kill():
     """ Check that we can kill a process, and its subprocess.
     """

IPython/genutils.py

@@ -979,6 +979,38 b' def get_home_dir():'
         else:
             raise HomeDirError,'support for your operating system not implemented.'
 
+
+def get_ipython_dir():
+    """Get the IPython directory for this platform and user.
+    
+    This uses the logic in `get_home_dir` to find the home directory
+    and the adds either .ipython or _ipython to the end of the path.
+    """
+    if os.name == 'posix':
+         ipdir_def = '.ipython'
+    else:
+         ipdir_def = '_ipython'
+    home_dir = get_home_dir()
+    ipdir = os.path.abspath(os.environ.get('IPYTHONDIR',
+                                           os.path.join(home_dir,ipdir_def)))
+    return ipdir
+
+def get_security_dir():
+    """Get the IPython security directory.
+    
+    This directory is the default location for all security related files,
+    including SSL/TLS certificates and FURL files.
+    
+    If the directory does not exist, it is created with 0700 permissions.
+    If it exists, permissions are set to 0700.
+    """
+    security_dir = os.path.join(get_ipython_dir(), 'security')
+    if not os.path.isdir(security_dir):
+        os.mkdir(security_dir, 0700)
+    else:
+        os.chmod(security_dir, 0700)
+    return security_dir
+        
 #****************************************************************************
 # strings and text
 

IPython/kernel/config/__init__.py

@@ -15,17 +15,15 b' __docformat__ = "restructuredtext en"'
 # Imports
 #-------------------------------------------------------------------------------
 
+from os.path import join as pjoin
+
 from IPython.external.configobj import ConfigObj
 from IPython.config.api import ConfigObjManager
-from IPython.config.cutils import get_ipython_dir
+from IPython.genutils import get_ipython_dir, get_security_dir
 
 default_kernel_config = ConfigObj()
 
-try:
-    ipython_dir = get_ipython_dir() + '/'
-except:
-    # This will defaults to the cwd
-    ipython_dir = ''
+security_dir = get_security_dir()
 
 #-------------------------------------------------------------------------------
 # Engine Configuration
@@ -33,7 +31,7 b' except:'
 
 engine_config = dict(
     logfile = '',    # Empty means log to stdout
-    furl_file = ipython_dir + 'ipcontroller-engine.furl'
+    furl_file = pjoin(security_dir, 'ipcontroller-engine.furl')
 )
 
 #-------------------------------------------------------------------------------
@@ -69,10 +67,10 b' controller_config = dict('
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
-        cert_file = ipython_dir + 'ipcontroller-engine.pem',
+        cert_file = pjoin(security_dir, 'ipcontroller-engine.pem'),
     ),
     engine_fc_interface = 'IPython.kernel.enginefc.IFCControllerBase',
-    engine_furl_file = ipython_dir + 'ipcontroller-engine.furl',
+    engine_furl_file = pjoin(security_dir, 'ipcontroller-engine.furl'),
     
     controller_interfaces = dict(
         # multiengine = dict(
@@ -83,12 +81,12 b' controller_config = dict('
         task = dict(
             controller_interface = 'IPython.kernel.task.ITaskController',
             fc_interface = 'IPython.kernel.taskfc.IFCTaskController',
-            furl_file = ipython_dir + 'ipcontroller-tc.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
             controller_interface = 'IPython.kernel.multiengine.IMultiEngine',
             fc_interface = 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine',
-            furl_file = ipython_dir + 'ipcontroller-mec.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     ),
 
@@ -97,7 +95,7 b' controller_config = dict('
         port = 0,        # 0 means pick a port for me
         location = '',    # Empty string means try to set automatically
         secure = True,
-        cert_file = ipython_dir + 'ipcontroller-client.pem'
+        cert_file = pjoin(security_dir, 'ipcontroller-client.pem')
     )
 )
 
@@ -108,10 +106,10 b' controller_config = dict('
 client_config = dict(
     client_interfaces = dict(
         task = dict(
-            furl_file = ipython_dir + 'ipcontroller-tc.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-tc.furl')
         ),
         multiengine = dict(
-            furl_file = ipython_dir + 'ipcontroller-mec.furl'
+            furl_file = pjoin(security_dir, 'ipcontroller-mec.furl')
         )
     )
 )

IPython/kernel/controllerservice.py

@@ -50,7 +50,7 b' from IPython.kernel.engineservice import \\'
     IEngineSerialized, \
     IEngineQueued
     
-from IPython.config import cutils
+from IPython.genutils import get_ipython_dir
 from IPython.kernel import codeutil
 
 #-------------------------------------------------------------------------------
@@ -170,7 +170,7 b' class ControllerService(object, service.Service):'
     
     def _getEngineInfoLogFile(self):
         # Store all logs inside the ipython directory
-        ipdir = cutils.get_ipython_dir()
+        ipdir = get_ipython_dir()
         pjoin = os.path.join
         logdir_base = pjoin(ipdir,'log')
         if not os.path.isdir(logdir_base):

IPython/kernel/scripts/ipcluster.py

@@ -93,7 +93,7 b' from subprocess import Popen,call'
 # IPython imports
 #---------------------------------------------------------------------------
 from IPython.tools import utils
-from IPython.config import cutils
+from IPython.genutils import get_ipython_dir
 
 #---------------------------------------------------------------------------
 # Normal code begins
@@ -180,7 +180,7 b' def clusterLocal(opt,arg):'
     """Start a cluster on the local machine."""
     
     # Store all logs inside the ipython directory
-    ipdir = cutils.get_ipython_dir()
+    ipdir = get_ipython_dir()
     pjoin = os.path.join
 
     logfile = opt.logfile
@@ -256,6 +256,24 b' def clusterLocal(opt,arg):'
 def clusterRemote(opt,arg):
     """Start a remote cluster over SSH"""
 
+    # B. Granger, 9/3/08
+    # The launching of a remote cluster using SSH and a clusterfile
+    # is broken.  Because it won't be fixed before the 0.9 release, 
+    # we are removing it.  For now, we just print a message to the 
+    # user and abort.
+    
+    print """The launching of a remote IPython cluster using SSL
+and a clusterfile has been removed in this release.  
+It has been broken for a while and we are in the process 
+of building a new process management system that will be 
+used to provide a more robust way of starting an IPython
+cluster.
+
+For now remote clusters have to be launched using ipcontroller
+and ipengine separately.
+    """
+    sys.exit(1)
+
     # Load the remote cluster configuration
     clConfig = {}
     execfile(opt.clusterfile,clConfig)
@@ -265,7 +283,7 b' def clusterRemote(opt,arg):'
     sshx = clConfig.get('sshx',os.environ.get('IPYTHON_SSHX','sshx'))
     
     # Store all logs inside the ipython directory
-    ipdir = cutils.get_ipython_dir()
+    ipdir = get_ipython_dir()
     pjoin = os.path.join
 
     logfile = opt.logfile
@@ -311,6 +329,11 b' def clusterRemote(opt,arg):'
 def main():
     """Main driver for the two big options: local or remote cluster."""
     
+    if sys.platform=='win32':
+        print """ipcluster does not work on Microsoft Windows.  Please start
+your IPython cluster using the ipcontroller and ipengine scripts."""
+        sys.exit(1)
+    
     opt,arg = parse_args()
 
     clusterfile = opt.clusterfile

IPython/kernel/scripts/ipengine.py

@@ -105,6 +105,7 b' def start_engine():'
     # register_engine to tell the controller we are ready to do work
     engine_connector = EngineConnector(tub_service)
     furl_file = kernel_config['engine']['furl_file']
+    log.msg("Using furl file: %s" % furl_file)
     d = engine_connector.connect_to_controller(engine_service, furl_file)
     d.addErrback(lambda _: reactor.stop())
     

IPython/kernel/tests/engineservicetest.py

@@ -245,7 +245,7 b' class IEngineSerializedTestCase(object):'
         self.assert_(es.IEngineSerialized.providedBy(self.engine))
         
     def testIEngineSerializedInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireCore."""
+        """Does self.engine have the methods and attributes in IEngineCore."""
         for m in list(es.IEngineSerialized):
             self.assert_(hasattr(self.engine, m))
        
@@ -288,7 +288,7 b' class IEngineQueuedTestCase(object):'
         self.assert_(es.IEngineQueued.providedBy(self.engine))
         
     def testIEngineQueuedInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireQueued."""
+        """Does self.engine have the methods and attributes in IEngineQueued."""
         for m in list(es.IEngineQueued):
             self.assert_(hasattr(self.engine, m))
             
@@ -326,7 +326,7 b' class IEnginePropertiesTestCase(object):'
         self.assert_(es.IEngineProperties.providedBy(self.engine))
     
     def testIEnginePropertiesInterfaceMethods(self):
-        """Does self.engine have the methods and attributes in IEngireProperties."""
+        """Does self.engine have the methods and attributes in IEngineProperties."""
         for m in list(es.IEngineProperties):
             self.assert_(hasattr(self.engine, m))
     

scripts/ipython_win_post_install.py

@@ -2,6 +2,7 b''
 """Windows-specific part of the installation"""
 
 import os, sys, shutil
+pjoin = os.path.join
 
 def mkshortcut(target,description,link_file,*args,**kw):
     """make a shortcut if it doesn't exist, and register its creation"""
@@ -11,69 +12,85 b' def mkshortcut(target,description,link_file,*args,**kw):'
 
 def install():
     """Routine to be run by the win32 installer with the -install switch."""
-
+    
     from IPython.Release import version
-
+    
     # Get some system constants
     prefix = sys.prefix
-    python = prefix + r'\python.exe'
-    # Lookup path to common startmenu ...
-    ip_dir = get_special_folder_path('CSIDL_COMMON_PROGRAMS') + r'\IPython'
+    python = pjoin(prefix, 'python.exe')
     
-    # Some usability warnings at installation time.  I don't want them at the
-    # top-level, so they don't appear if the user is uninstalling.
-    try:
-        import ctypes
-    except ImportError:
-        print ('To take full advantage of IPython, you need ctypes from:\n'
-               'http://sourceforge.net/projects/ctypes')
-
-    try:
-        import win32con
-    except ImportError:
-        print ('To take full advantage of IPython, you need pywin32 from:\n'
-               'http://starship.python.net/crew/mhammond/win32/Downloads.html')
-
-    try:
-        import readline
-    except ImportError:
-        print ('To take full advantage of IPython, you need readline from:\n'
-               'https://launchpad.net/pyreadline')
-
-    ipybase = '"' + prefix + r'\scripts\ipython"'
+    # Lookup path to common startmenu ...
+    ip_start_menu = pjoin(get_special_folder_path('CSIDL_COMMON_PROGRAMS'), 'IPython')
     # Create IPython entry ...
-    if not os.path.isdir(ip_dir):
-        os.mkdir(ip_dir)
-        directory_created(ip_dir)
-
-    # Create program shortcuts ...
-    f = ip_dir + r'\IPython.lnk'
-    a = ipybase
-    mkshortcut(python,'IPython',f,a)
-
-    f = ip_dir + r'\pysh.lnk'
-    a = ipybase+' -p sh'
-    mkshortcut(python,'IPython (command prompt mode)',f,a)
-
-    f = ip_dir + r'\pylab.lnk'
-    a = ipybase+' -pylab'
-    mkshortcut(python,'IPython (PyLab mode)',f,a)
-
-    f = ip_dir + r'\scipy.lnk'
-    a = ipybase+' -pylab -p scipy'
-    mkshortcut(python,'IPython (scipy profile)',f,a)
-
+    if not os.path.isdir(ip_start_menu):
+        os.mkdir(ip_start_menu)
+        directory_created(ip_start_menu)
+    
+    # Create .py and .bat files to make things available from
+    # the Windows command line.  Thanks to the Twisted project
+    # for this logic!
+    programs = [
+        'ipython',
+        'iptest',
+        'ipcontroller',
+        'ipengine',
+        'ipcluster',
+        'ipythonx',
+        'ipython-wx',
+        'irunner'
+    ]
+    scripts = pjoin(prefix,'scripts')
+    for program in programs:
+        raw = pjoin(scripts, program)
+        bat = raw + '.bat'
+        py = raw + '.py'
+        # Create .py versions of the scripts
+        shutil.copy(raw, py)
+        # Create .bat files for each of the scripts
+        bat_file = file(bat,'w')
+        bat_file.write("@%s %s %%*" % (python, py))
+        bat_file.close()
+            
+    # Now move onto setting the Start Menu up
+    ipybase = pjoin(scripts, 'ipython')
+
+    link = pjoin(ip_start_menu, 'IPython.lnk')
+    cmd = '"%s"' % ipybase
+    mkshortcut(python,'IPython',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'pysh.lnk')
+    cmd = '"%s" -p sh' % ipybase
+    mkshortcut(python,'IPython (command prompt mode)',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'pylab.lnk')
+    cmd = '"%s" -pylab' % ipybase
+    mkshortcut(python,'IPython (PyLab mode)',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'scipy.lnk')
+    cmd = '"%s" -pylab -p scipy' % ipybase
+    mkshortcut(python,'IPython (scipy profile)',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'IPython test suite.lnk')
+    cmd = '"%s" -vv' % pjoin(scripts, 'iptest')
+    mkshortcut(python,'Run the IPython test suite',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'ipcontroller.lnk')
+    cmd = '"%s" -xy' % pjoin(scripts, 'ipcontroller')
+    mkshortcut(python,'IPython controller',link,cmd)
+    
+    link = pjoin(ip_start_menu, 'ipengine.lnk')
+    cmd = '"%s"' % pjoin(scripts, 'ipengine')
+    mkshortcut(python,'IPython engine',link,cmd)
+    
     # Create documentation shortcuts ...
     t = prefix + r'\share\doc\ipython\manual\ipython.pdf'
-    f = ip_dir + r'\Manual in PDF.lnk'
+    f = ip_start_menu + r'\Manual in PDF.lnk'
     mkshortcut(t,r'IPython Manual - PDF-Format',f)
-
+    
     t = prefix + r'\share\doc\ipython\manual\html\index.html'
-    f = ip_dir + r'\Manual in HTML.lnk'
+    f = ip_start_menu + r'\Manual in HTML.lnk'
     mkshortcut(t,'IPython Manual - HTML-Format',f)
-
-    # make ipython.py
-    shutil.copy(prefix + r'\scripts\ipython', prefix + r'\scripts\ipython.py')
+    
     
 def remove():
     """Routine to be run by the win32 installer with the -remove switch."""

setupbase.py

@@ -115,6 +115,7 b' def find_packages():'
     add_package(packages, 'kernel', config=True, tests=True, scripts=True)
     add_package(packages, 'kernel.core', config=True, tests=True)
     add_package(packages, 'testing', tests=True)
+    add_package(packages, 'tests')
     add_package(packages, 'testing.plugin', tests=False)
     add_package(packages, 'tools', tests=True)
     add_package(packages, 'UserConfig')