Merge pull request #2868 from takluyver/import-performance

Import performance

Defer various imports for a small reduction in startup time.

IPython.lib.__init__ previously loaded IPython.lib.inputhook to export some of its functions. This meant that importing anything from IPython.lib also loaded inputhook.

For now, I've just removed that import, but that is an API change, because the functions are no longer accessible as e.g. IPython.lib.enable_qt4

Added a note about the API change.

Timing command, borrowed from Openoffice to simulate cold start:

$ sync ; echo 3 | sudo tee /proc/sys/vm/drop_caches; time ipython3 -c pass

Results with this branch: 7.68, 7.64, 7.35, 7.38 s 'real'
Results with master: 8.17, 8.04, 7.81, 7.77

IPython/core/alias.py

@@ -30,7 +30,6 b' from IPython.config.configurable import Configurable'
 from IPython.core.splitinput import split_user_input
 
 from IPython.utils.traitlets import List, Instance
-from IPython.utils.autoattr import auto_attr
 from IPython.utils.warn import warn, error
 
 #-----------------------------------------------------------------------------

IPython/core/completer.py

@@ -74,7 +74,6 b' import itertools'
 import keyword
 import os
 import re
-import shlex
 import sys
 
 from IPython.config.configurable import Configurable

IPython/core/completerlib.py

@@ -30,7 +30,6 b' from zipimport import zipimporter'
 # Our own imports
 from IPython.core.completer import expand_user, compress_user
 from IPython.core.error import TryNext
-from IPython.utils import py3compat
 from IPython.utils._process_common import arg_split
 
 # FIXME: this should be pulled in with the right call via the component system

IPython/core/debugger.py

@@ -33,7 +33,7 b' import sys'
 
 from IPython.utils import PyColorize, ulinecache
 from IPython.core import ipapi
-from IPython.utils import coloransi, io, openpy, py3compat
+from IPython.utils import coloransi, io, py3compat
 from IPython.core.excolors import exception_colors
 
 # See if we can use pydb.

IPython/core/display.py

@@ -19,8 +19,6 b' Authors:'
 
 from __future__ import print_function
 
-from xml.dom import minidom
-
 from .displaypub import (
     publish_pretty, publish_html,
     publish_latex, publish_svg,
@@ -330,6 +328,7 b' class SVG(DisplayObject):'
             self._data = None
             return
         # parse into dom object
+        from xml.dom import minidom
         x = minidom.parseString(svg)
         # get svg tag (should be 1)
         found_svg = x.getElementsByTagName('svg')

IPython/core/displayhook.py

@@ -30,7 +30,7 b' import sys'
 
 from IPython.config.configurable import Configurable
 from IPython.utils import io
-from IPython.utils.traitlets import Instance, List
+from IPython.utils.traitlets import Instance
 from IPython.utils.warn import warn
 
 #-----------------------------------------------------------------------------

IPython/core/extensions.py

@@ -20,8 +20,6 b' Authors:'
 import os
 from shutil import copyfile
 import sys
-from urllib import urlretrieve
-from urlparse import urlparse
 
 from IPython.core.error import UsageError
 from IPython.config.configurable import Configurable
@@ -171,6 +169,8 b' class ExtensionManager(Configurable):'
             src_filename = os.path.basename(url)
             copy = copyfile
         else:
+            from urllib import urlretrieve  # Deferred imports
+            from urlparse import urlparse
             src_filename = urlparse(url).path.split('/')[-1]
             copy = urlretrieve
             

IPython/core/interactiveshell.py

@@ -28,7 +28,6 b' import runpy'
 import sys
 import tempfile
 import types
-import urllib
 from io import open as io_open
 
 from IPython.config.configurable import SingletonConfigurable
@@ -2989,7 +2988,8 b' class InteractiveShell(SingletonConfigurable):'
                 return openpy.read_py_url(utarget, skip_encoding_cookie=skip_encoding_cookie)
         except UnicodeDecodeError:
             if not py_only :
-                response = urllib.urlopen(target)
+                from urllib import urlopen  # Deferred import
+                response = urlopen(target)
                 return response.read().decode('latin1')
             raise ValueError(("'%s' seem to be unreadable.") % utarget)
 

IPython/core/macro.py

@@ -8,7 +8,6 b''
 #*****************************************************************************
 
 import re
-import sys
 
 from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING

IPython/core/magic.py

@@ -31,7 +31,7 b' from IPython.utils.ipstruct import Struct'
 from IPython.utils.process import arg_split
 from IPython.utils.text import dedent
 from IPython.utils.traitlets import Bool, Dict, Instance, MetaHasTraits
-from IPython.utils.warn import error, warn
+from IPython.utils.warn import error
 
 #-----------------------------------------------------------------------------
 # Globals

IPython/core/magics/code.py

@@ -15,11 +15,9 b''
 # Stdlib
 import inspect
 import io
-import json
 import os
 import re
 import sys
-from urllib2 import urlopen
 
 # Our own packages
 from IPython.core.error import TryNext, StdinNotImplementedError, UsageError
@@ -27,7 +25,6 b' from IPython.core.macro import Macro'
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.core.oinspect import find_file, find_source_lines
 from IPython.testing.skipdoctest import skip_doctest
-from IPython.utils import openpy
 from IPython.utils import py3compat
 from IPython.utils.contexts import preserve_keys
 from IPython.utils.io import file_read
@@ -141,6 +138,8 b' class CodeMagics(Magics):'
             print e.args[0]
             return
 
+        from urllib2 import urlopen  # Deferred import
+        import json
         post_data = json.dumps({
           "description": opts.get('d', "Pasted from IPython"),
           "public": True,

IPython/core/magics/display.py

@@ -14,7 +14,7 b''
 # Our own packages
 from IPython.core.display import display, Javascript, Latex, SVG
 from IPython.core.magic import  (
-    Magics, magics_class, line_magic, cell_magic
+    Magics, magics_class, cell_magic
 )
 
 #-----------------------------------------------------------------------------

IPython/core/magics/history.py

@@ -16,7 +16,6 b' from __future__ import print_function'
 # Stdlib
 import os
 from io import open as io_open
-from IPython.external.argparse import Action
 
 # Our own packages
 from IPython.core.error import StdinNotImplementedError

IPython/core/magics/osm.py

@@ -26,14 +26,14 b' from pprint import pformat'
 from IPython.core import magic_arguments
 from IPython.core import oinspect
 from IPython.core import page
-from IPython.core.error import UsageError, StdinNotImplementedError
+from IPython.core.error import UsageError
 from IPython.core.magic import  (
     Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
 )
 from IPython.testing.skipdoctest import skip_doctest
-from IPython.utils.io import file_read, nlprint
+from IPython.utils.io import nlprint
 from IPython.utils.openpy import source_to_unicode
-from IPython.utils.path import get_py_filename, unquote_filename
+from IPython.utils.path import unquote_filename
 from IPython.utils.process import abbrev_cwd
 from IPython.utils.terminal import set_term_title
 #-----------------------------------------------------------------------------

IPython/core/magics/script.py

@@ -14,7 +14,6 b''
 # Stdlib
 import errno
 import os
-import re
 import sys
 import signal
 import time
@@ -24,12 +23,10 b' import atexit'
 # Our own packages
 from IPython.config.configurable import Configurable
 from IPython.core import magic_arguments
-from IPython.core.error import UsageError
 from IPython.core.magic import  (
     Magics, magics_class, line_magic, cell_magic
 )
 from IPython.lib.backgroundjobs import BackgroundJobManager
-from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import py3compat
 from IPython.utils.process import arg_split
 from IPython.utils.traitlets import List, Dict

IPython/core/page.py

@@ -29,7 +29,6 b' from __future__ import print_function'
 
 import os
 import re
-import subprocess
 import sys
 import tempfile
 
@@ -37,7 +36,6 b' from io import UnsupportedOperation'
 
 from IPython.core import ipapi
 from IPython.core.error import TryNext
-from IPython.utils.cursesimport import use_curses
 from IPython.utils.data import chop
 from IPython.utils import io
 from IPython.utils.process import system
@@ -73,61 +71,66 b' def page_dumb(strng, start=0, screen_lines=25):'
                 last_escape = esc_list[-1]
         print(last_escape + os.linesep.join(screens[-1]), file=io.stdout)
 
-def _detect_screen_size(use_curses, screen_lines_def):
+def _detect_screen_size(screen_lines_def):
     """Attempt to work out the number of lines on the screen.
 
     This is called by page(). It can raise an error (e.g. when run in the
     test suite), so it's separated out so it can easily be called in a try block.
     """
     TERM = os.environ.get('TERM',None)
-    if (TERM=='xterm' or TERM=='xterm-color') and sys.platform != 'sunos5':
-        local_use_curses = use_curses
-    else:
+    if not((TERM=='xterm' or TERM=='xterm-color') and sys.platform != 'sunos5'):
         # curses causes problems on many terminals other than xterm, and
         # some termios calls lock up on Sun OS5.
-        local_use_curses = False
-    if local_use_curses:
+        return screen_lines_def
+    
+    try:
         import termios
         import curses
-        # There is a bug in curses, where *sometimes* it fails to properly
-        # initialize, and then after the endwin() call is made, the
-        # terminal is left in an unusable state.  Rather than trying to
-        # check everytime for this (by requesting and comparing termios
-        # flags each time), we just save the initial terminal state and
-        # unconditionally reset it every time.  It's cheaper than making
-        # the checks.
-        term_flags = termios.tcgetattr(sys.stdout)
-
-        # Curses modifies the stdout buffer size by default, which messes
-        # up Python's normal stdout buffering.  This would manifest itself
-        # to IPython users as delayed printing on stdout after having used
-        # the pager.
-        #
-        # We can prevent this by manually setting the NCURSES_NO_SETBUF
-        # environment variable.  For more details, see:
-        # http://bugs.python.org/issue10144
-        NCURSES_NO_SETBUF = os.environ.get('NCURSES_NO_SETBUF', None)
-        os.environ['NCURSES_NO_SETBUF'] = ''
-
-        # Proceed with curses initialization
+    except ImportError:
+        return screen_lines_def
+    
+    # There is a bug in curses, where *sometimes* it fails to properly
+    # initialize, and then after the endwin() call is made, the
+    # terminal is left in an unusable state.  Rather than trying to
+    # check everytime for this (by requesting and comparing termios
+    # flags each time), we just save the initial terminal state and
+    # unconditionally reset it every time.  It's cheaper than making
+    # the checks.
+    term_flags = termios.tcgetattr(sys.stdout)
+
+    # Curses modifies the stdout buffer size by default, which messes
+    # up Python's normal stdout buffering.  This would manifest itself
+    # to IPython users as delayed printing on stdout after having used
+    # the pager.
+    #
+    # We can prevent this by manually setting the NCURSES_NO_SETBUF
+    # environment variable.  For more details, see:
+    # http://bugs.python.org/issue10144
+    NCURSES_NO_SETBUF = os.environ.get('NCURSES_NO_SETBUF', None)
+    os.environ['NCURSES_NO_SETBUF'] = ''
+
+    # Proceed with curses initialization
+    try:
         scr = curses.initscr()
-        screen_lines_real,screen_cols = scr.getmaxyx()
-        curses.endwin()
+    except AttributeError:
+        # Curses on Solaris may not be complete, so we can't use it there
+        return screen_lines_def
+    
+    screen_lines_real,screen_cols = scr.getmaxyx()
+    curses.endwin()
 
-        # Restore environment
-        if NCURSES_NO_SETBUF is None:
-            del os.environ['NCURSES_NO_SETBUF']
-        else:
-            os.environ['NCURSES_NO_SETBUF'] = NCURSES_NO_SETBUF
-
-        # Restore terminal state in case endwin() didn't.
-        termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
-        # Now we have what we needed: the screen size in rows/columns
-        return screen_lines_real
-        #print '***Screen size:',screen_lines_real,'lines x',\
-        #screen_cols,'columns.' # dbg
+    # Restore environment
+    if NCURSES_NO_SETBUF is None:
+        del os.environ['NCURSES_NO_SETBUF']
     else:
-        return screen_lines_def
+        os.environ['NCURSES_NO_SETBUF'] = NCURSES_NO_SETBUF
+
+    # Restore terminal state in case endwin() didn't.
+    termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
+    # Now we have what we needed: the screen size in rows/columns
+    return screen_lines_real
+    #print '***Screen size:',screen_lines_real,'lines x',\
+    #screen_cols,'columns.' # dbg
 
 def page(strng, start=0, screen_lines=0, pager_cmd=None):
     """Print a string, piping through a pager after a certain length.
@@ -184,7 +187,7 b' def page(strng, start=0, screen_lines=0, pager_cmd=None):'
     # auto-determine screen size
     if screen_lines <= 0:
         try:
-            screen_lines += _detect_screen_size(use_curses, screen_lines_def)
+            screen_lines += _detect_screen_size(screen_lines_def)
         except (TypeError, UnsupportedOperation):
             print(str_toprint, file=io.stdout)
             return

IPython/core/prefilter.py

@@ -24,11 +24,8 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-import __builtin__
-import codeop
 import re
 
-from IPython.core.alias import AliasManager
 from IPython.core.autocall import IPyAutocall
 from IPython.config.configurable import Configurable
 from IPython.core.inputsplitter import (
@@ -42,13 +39,11 b' from IPython.core.inputsplitter import ('
     ESC_PAREN,
 )
 from IPython.core.macro import Macro
-from IPython.core.splitinput import split_user_input, LineInfo
-from IPython.core import page
+from IPython.core.splitinput import LineInfo
 
 from IPython.utils.traitlets import (
-    List, Integer, Any, Unicode, CBool, Bool, Instance, CRegExp
+    List, Integer, Unicode, CBool, Bool, Instance, CRegExp
 )
-from IPython.utils.autoattr import auto_attr
 
 #-----------------------------------------------------------------------------
 # Global utilities, errors and constants

IPython/core/profileapp.py

@@ -21,12 +21,11 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-import logging
 import os
 
-from IPython.config.application import Application, boolean_flag
+from IPython.config.application import Application
 from IPython.core.application import (
-    BaseIPythonApplication, base_flags, base_aliases
+    BaseIPythonApplication, base_flags
 )
 from IPython.core.profiledir import ProfileDir
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir

IPython/core/profiledir.py

@@ -23,12 +23,10 b' Authors:'
 
 import os
 import shutil
-import sys
 
 from IPython.config.configurable import LoggingConfigurable
-from IPython.config.loader import Config
 from IPython.utils.path import get_ipython_package_dir, expand_path
-from IPython.utils.traitlets import List, Unicode, Bool
+from IPython.utils.traitlets import Unicode, Bool
 
 #-----------------------------------------------------------------------------
 # Classes and functions

IPython/core/ultratb.py

@@ -109,7 +109,6 b' from IPython.utils import py3compat'
 from IPython.utils import pyfile
 from IPython.utils import ulinecache
 from IPython.utils.data import uniq_stable
-from IPython.utils.openpy import read_py_file
 from IPython.utils.warn import info, error
 
 # Globals

IPython/frontend/terminal/interactiveshell.py

@@ -358,8 +358,13 b' class TerminalInteractiveShell(InteractiveShell):'
     using_paste_magics = CBool(False)
 
     # In the terminal, GUI control is done via PyOS_InputHook
-    from IPython.lib.inputhook import enable_gui
-    enable_gui = staticmethod(enable_gui)
+    @staticmethod
+    def enable_gui(gui=None, app=None):
+        """Switch amongst GUI input hooks by name.
+        """
+        # Deferred import
+        from IPython.lib.inputhook import enable_gui as real_enable_gui
+        return real_enable_gui(gui, app)
     
     def __init__(self, config=None, ipython_dir=None, profile_dir=None,
                  user_ns=None, user_module=None, custom_exceptions=((),None),

IPython/frontend/terminal/ipapp.py

@@ -48,7 +48,6 b' from IPython.core.shellapp import ('
     InteractiveShellApp, shell_flags, shell_aliases
 )
 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
-from IPython.lib import inputhook
 from IPython.utils import warn
 from IPython.utils.path import get_ipython_dir, check_for_old_config
 from IPython.utils.traitlets import (

IPython/lib/__init__.py

@@ -14,18 +14,6 b' Extra capabilities for IPython'
 # Imports
 #-----------------------------------------------------------------------------
 
-from IPython.lib.inputhook import (
-    enable_wx, disable_wx,
-    enable_gtk, disable_gtk,
-    enable_qt4, disable_qt4,
-    enable_tk, disable_tk,
-    enable_glut, disable_glut,
-    enable_pyglet, disable_pyglet,
-    enable_gtk3, disable_gtk3,
-    set_inputhook, clear_inputhook,
-    current_gui
-)
-
 from IPython.lib.security import passwd
 
 #-----------------------------------------------------------------------------

IPython/lib/backgroundjobs.py

@@ -34,7 +34,7 b' import sys'
 import threading
 
 from IPython.core.ultratb import AutoFormattedTB
-from IPython.utils.warn import warn, error
+from IPython.utils.warn import error
 
 
 class BackgroundJobManager(object):

IPython/lib/clipboard.py

@@ -2,7 +2,6 b''
 """
 
 import subprocess
-import sys
 
 from IPython.core.error import TryNext
 import IPython.utils.py3compat as py3compat

IPython/lib/demo.py

@@ -183,9 +183,8 b' import re'
 import shlex
 import sys
 
-from IPython.utils.PyColorize import Parser
 from IPython.utils import io
-from IPython.utils.io import file_read, file_readlines
+from IPython.utils.io import file_read
 from IPython.utils.text import marquee
 from IPython.utils import openpy
 __all__ = ['Demo','IPythonDemo','LineDemo','IPythonLineDemo','DemoError']

IPython/lib/display.py

@@ -2,8 +2,6 b''
 
 Authors : MinRK, gregcaporaso, dannystaple
 """
-import urllib
-
 from os.path import exists, isfile, splitext, abspath, join, isdir
 from os import walk, sep
 
@@ -41,7 +39,8 b' class YouTubeVideo(object):'
     def _repr_html_(self):
         """return YouTube embed iframe for this video id"""
         if self.params:
-            params = "?" + urllib.urlencode(self.params)
+            from urllib import urlencode  # Deferred import
+            params = "?" + urlencode(self.params)
         else:
             params = ""
         return """

IPython/lib/inputhookwx.py

@@ -17,9 +17,7 b' Authors: Robin Dunn, Brian Granger, Ondrej Certik'
 # Imports
 #-----------------------------------------------------------------------------
 
-import os
 import signal
-import sys
 import time
 from timeit import default_timer as clock
 import wx

IPython/lib/latextools.py

@@ -26,7 +26,7 b' import subprocess'
 
 from IPython.utils.process import find_cmd, FindCmdError
 from IPython.config.configurable import SingletonConfigurable
-from IPython.utils.traitlets import Instance, List, CBool, CUnicode
+from IPython.utils.traitlets import List, CBool, CUnicode
 from IPython.utils.py3compat import bytes_to_str
 
 #-----------------------------------------------------------------------------

IPython/utils/PyColorize.py

@@ -41,7 +41,6 b" _scheme_default = 'Linux'"
 import StringIO
 import keyword
 import os
-import optparse
 import sys
 import token
 import tokenize
@@ -263,6 +262,7 b' def main(argv=None):'
 Colorize a python file or stdin using ANSI color escapes and print to stdout.
 If no filename is given, or if filename is -, read standard input."""
 
+    import optparse
     parser = optparse.OptionParser(usage=usage_msg)
     newopt = parser.add_option
     newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store',

IPython/utils/_process_posix.py

@@ -24,7 +24,6 b' from IPython.external import pexpect'
 # Our own
 from .autoattr import auto_attr
 from ._process_common import getoutput, arg_split
-from IPython.utils import text
 from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING
 

IPython/utils/_process_win32.py

@@ -28,7 +28,6 b' from subprocess import STDOUT'
 # our own imports
 from ._process_common import read_no_interrupt, process_handler, arg_split as py_arg_split
 from . import py3compat
-from . import text
 from .encoding import DEFAULT_ENCODING
 
 #-----------------------------------------------------------------------------

IPython/utils/jsonutil.py

@@ -13,7 +13,6 b''
 # stdlib
 import math
 import re
-import sys
 import types
 from datetime import datetime
 
@@ -26,7 +25,6 b' except ImportError:'
 
 from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING
-from IPython.utils import text
 next_attr_name = '__next__' if py3compat.PY3 else 'next'
 
 #-----------------------------------------------------------------------------

IPython/utils/openpy.py

@@ -9,7 +9,6 b' from __future__ import absolute_import'
 import io
 from io import TextIOWrapper, BytesIO
 import re
-import urllib
 
 cookie_re = re.compile(ur"coding[:=]\s*([-\w.]+)", re.UNICODE)
 cookie_comment_re = re.compile(ur"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE)
@@ -205,7 +204,8 b" def read_py_url(url, errors='replace', skip_encoding_cookie=True):"
     -------
     A unicode string containing the contents of the file.
     """
-    response = urllib.urlopen(url)
+    from urllib import urlopen  # Deferred import for faster start
+    response = urlopen(url)
     buffer = io.BytesIO(response.read())
     return source_to_unicode(buffer, errors, skip_encoding_cookie)
 

IPython/utils/process.py

@@ -28,7 +28,6 b' else:'
 
 
 from ._process_common import getoutputerror
-from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Code

IPython/utils/rlineimpl.py

@@ -9,14 +9,9 b' In addition to normal readline stuff, this module provides have_readline'
 boolean and _outputfile variable used in IPython.utils.
 """
 
-import os
-import re
 import sys
-import time
 import warnings
 
-from subprocess import Popen, PIPE
-
 if sys.platform == 'darwin':
     # dirty trick, to skip the system readline, because pip-installed readline
     # will never be found on OSX, since lib-dynload always comes ahead of site-packages

IPython/utils/sysinfo.py

@@ -20,8 +20,6 b' import pprint'
 import sys
 import subprocess
 
-from ConfigParser import ConfigParser
-
 from IPython.core import release
 from IPython.utils import py3compat, _sysinfo, encoding
 

IPython/utils/text.py

@@ -23,7 +23,6 b' import __main__'
 
 import os
 import re
-import shutil
 import sys
 import textwrap
 from string import Formatter
@@ -403,31 +402,6 b' def indent(instr,nspaces=4, ntabs=0, flatten=False):'
     else:
         return outstr
 
-def native_line_ends(filename,backup=1):
-    """Convert (in-place) a file to line-ends native to the current OS.
-
-    If the optional backup argument is given as false, no backup of the
-    original file is left.  """
-
-    backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}
-
-    bak_filename = filename + backup_suffixes[os.name]
-
-    original = open(filename).read()
-    shutil.copy2(filename,bak_filename)
-    try:
-        new = open(filename,'wb')
-        new.write(os.linesep.join(original.splitlines()))
-        new.write(os.linesep) # ALWAYS put an eol at the end of the file
-        new.close()
-    except:
-        os.rename(bak_filename,filename)
-    if not backup:
-        try:
-            os.remove(bak_filename)
-        except:
-            pass
-
 
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings

docs/source/whatsnew/development.txt

@@ -32,3 +32,7 b' Backwards incompatible changes'
   was duplicated in :mod:`IPython.core.inputsplitter`, and the latter version
   was already what IPython relied on. A new API to transform input will be ready
   before release.
+* Functions from :mod:`IPython.lib.inputhook` to control integration with GUI
+  event loops are no longer exposed in the top level of :mod:`IPython.lib`.
+  Code calling these should make sure to import them from
+  :mod:`IPython.lib.inputhook`.