Merge pull request #3275 from minrk/submodule-hooks

improve submodule messages / git hooks

IPython/utils/submodule.py

@@ -0,0 +1,92 b''
+"""utilities for checking submodule status"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2013  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import os
+import subprocess
+import sys
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+pjoin = os.path.join
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+def ipython_parent():
+    """return IPython's parent (i.e. root if run from git)"""
+    from IPython.utils.path import get_ipython_package_dir
+    return os.path.abspath(os.path.dirname(get_ipython_package_dir()))
+
+def ipython_submodules(root):
+    """return IPython submodules relative to root"""
+    return [
+        pjoin(root, 'IPython', 'frontend', 'html', 'notebook', 'static', 'components'),
+    ]
+
+def is_repo(d):
+    """is d a git repo?"""
+    return os.path.exists(pjoin(d, '.git'))
+
+def check_submodule_status(root=None):
+    """check submodule status
+
+    Has three return values:
+
+    'missing' - submodules are absent
+    'unclean' - submodules have unstaged changes
+    'clean' - all submodules are up to date
+    """
+
+    if hasattr(sys, "frozen"):
+      # frozen via py2exe or similar, don't bother
+      return 'clean'
+
+    if not root:
+        root = ipython_parent()
+
+    submodules = ipython_submodules(root)
+
+    for submodule in submodules:
+        if not os.path.exists(submodule):
+            return 'missing'
+
+    if not is_repo(root):
+        # not in git, assume clean
+        return 'clean'
+
+    # check with git submodule status
+    proc = subprocess.Popen('git submodule status',
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE,
+                            shell=True,
+                            cwd=root,
+    )
+    status, _ = proc.communicate()
+    status = status.decode("ascii")
+
+    for line in status.splitlines():
+        if status.startswith('-'):
+            return 'missing'
+        elif status.startswith('+'):
+            return 'unclean'
+
+    return 'clean'
+
+def update_submodules(repo_dir):
+    """update submodules in a repo"""
+    subprocess.check_call("git submodule init", cwd=repo_dir, shell=True)
+    subprocess.check_call("git submodule update --recursive", cwd=repo_dir, shell=True)
+

git-hooks/README.md

@@ -0,0 +1,13 b''
+git hooks for IPython
+
+add these to your `.git/hooks`
+
+For now, we just have `post-checkout` and `post-merge`,
+both of which just update submodules,
+so make sure that you have a fully synced repo whenever you checkout or pull.
+
+To use these hooks, you can symlink or copy them to your `.git/hooks` directory.
+
+    ln -s ../../git-hooks/post-checkout .git/hooks/post-checkout
+    ln -s ../../git-hooks/post-merge .git/hooks/post-merge
+

git-hooks/post-checkout

@@ -0,0 +1,4 b''
+#!/bin/sh
+
+git submodule init
+git submodule update

git-hooks/post-merge

@@ -0,0 +1,4 b''
+#!/bin/sh
+
+git submodule init
+git submodule update

IPython/config/application.py

@@ -95,6 +95,26 b' def catch_config_error(method, app, *args, **kwargs):'
 class ApplicationError(Exception):
     pass
 
+class LevelFormatter(logging.Formatter):
+    """Formatter with additional `highlevel` record
+    
+    This field is empty if log level is less than highlevel_limit,
+    otherwise it is formatted with self.highlevel_format.
+    
+    Useful for adding 'WARNING' to warning messages,
+    without adding 'INFO' to info, etc.
+    """
+    highlevel_limit = logging.WARN
+    highlevel_format = " %(levelname)s |"
+    
+    def format(self, record):
+        if record.levelno >= self.highlevel_limit:
+            record.highlevel = self.highlevel_format % record.__dict__
+        else:
+            record.highlevel = ""
+        
+        return super(LevelFormatter, self).format(record)
+            
 
 class Application(SingletonConfigurable):
     """A singleton application with full configuration support."""
@@ -133,13 +153,19 b' class Application(SingletonConfigurable):'
             self.log_level = new
         self.log.setLevel(new)
     
-    log_format = Unicode("[%(name)s] %(message)s", config=True,
+    log_datefmt = Unicode("%Y-%m-%d %H:%M:%S", config=True,
+        help="The date format used by logging formatters for %(asctime)s"
+    )
+    def _log_datefmt_changed(self, name, old, new):
+        self._log_format_changed()
+    
+    log_format = Unicode("[%(name)s]%(highlevel)s %(message)s", config=True,
         help="The Logging format template",
     )
     def _log_format_changed(self, name, old, new):
         """Change the log formatter when log_format is set."""
         _log_handler = self.log.handlers[0]
-        _log_formatter = logging.Formatter(new)
+        _log_formatter = LevelFormatter(new, datefmt=self._log_datefmt)
         _log_handler.setFormatter(_log_formatter)
 
     log = Instance(logging.Logger)
@@ -167,7 +193,7 b' class Application(SingletonConfigurable):'
             _log_handler = logging.StreamHandler(open(os.devnull, 'w'))
         else:
             _log_handler = logging.StreamHandler()
-        _log_formatter = logging.Formatter(self.log_format)
+        _log_formatter = LevelFormatter(self.log_format, datefmt=self.log_datefmt)
         _log_handler.setFormatter(_log_formatter)
         log.addHandler(_log_handler)
         return log

IPython/frontend/html/notebook/notebookapp.py

@@ -6,7 +6,7 b' Authors:'
 * Brian Granger
 """
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2011  The IPython Development Team
+#  Copyright (C) 2013  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -90,6 +90,7 b' from IPython.kernel.zmq.kernelapp import ('
 )
 from IPython.utils.importstring import import_item
 from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils import submodule
 from IPython.utils.traitlets import (
     Dict, Unicode, Integer, List, Enum, Bool,
     DottedObjectName
@@ -323,7 +324,7 b' class NotebookApp(BaseIPythonApplication):'
 
     def _log_format_default(self):
         """override default log format to include time"""
-        return u"%(asctime)s.%(msecs).03d [%(name)s] %(message)s"
+        return u"%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s"
 
     # create requested profiles by default, if they don't exist:
     auto_create = Bool(True)
@@ -537,10 +538,6 b' class NotebookApp(BaseIPythonApplication):'
         # and all of its ancenstors until propagate is set to False.
         self.log.propagate = False
         
-        # set the date format
-        formatter = logging.Formatter(self.log_format, datefmt="%Y-%m-%d %H:%M:%S")
-        self.log.handlers[0].setFormatter(formatter)
-        
         # hook up tornado 3's loggers to our app handlers
         for name in ('access', 'application', 'general'):
             logging.getLogger('tornado.%s' % name).handlers = self.log.handlers
@@ -673,11 +670,23 b' class NotebookApp(BaseIPythonApplication):'
     def _signal_info(self, sig, frame):
         print self.notebook_info()
     
+    def init_components(self):
+        """Check the components submodule, and warn if it's unclean"""
+        status = submodule.check_submodule_status()
+        if status == 'missing':
+            self.log.warn("components submodule missing, running `git submodule update`")
+            submodule.update_submodules(submodule.ipython_parent())
+        elif status == 'unclean':
+            self.log.warn("components submodule unclean, you may see 404s on static/components")
+            self.log.warn("run `setup.py submodule` or `git submodule update` to update")
+            
+    
     @catch_config_error
     def initialize(self, argv=None):
         self.init_logging()
         super(NotebookApp, self).initialize(argv)
         self.init_configurables()
+        self.init_components()
         self.init_webapp()
         self.init_signal()
 

IPython/kernel/zmq/kernelapp.py

@@ -307,10 +307,6 b' class IPKernelApp(BaseIPythonApplication, InteractiveShellApp):'
         self.hb_port = self.heartbeat.port
         self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port)
         self.heartbeat.start()
-
-        # Helper to make it easier to connect to an existing kernel.
-        # set log-level to critical, to make sure it is output
-        self.log.critical("To connect another client to this kernel, use:")
     
     def log_connection_info(self):
         """display connection info, and store ports"""
@@ -323,8 +319,20 b' class IPKernelApp(BaseIPythonApplication, InteractiveShellApp):'
                 tail += " --profile %s" % self.profile
         else:
             tail = self.connection_file
-        self.log.critical("--existing %s", tail)
-
+        lines = [
+            "To connect another client to this kernel, use:",
+            "    --existing %s" % tail,
+        ]
+        # log connection info
+        # info-level, so often not shown.
+        # frontends should use the %connect_info magic
+        # to see the connection info
+        for line in lines:
+            self.log.info(line)
+        # also raw print to the terminal if no parent (`ipython kernel`)
+        if not self.parent:
+            for line in lines:
+                io.rprint(line)
 
         self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
                                 stdin=self.stdin_port, hb=self.hb_port,

IPython/parallel/apps/baseapp.py

@@ -29,7 +29,7 b' import sys'
 
 from subprocess import Popen, PIPE
 
-from IPython.config.application import catch_config_error
+from IPython.config.application import catch_config_error, LevelFormatter
 from IPython.core import release
 from IPython.core.crashhandler import CrashHandler
 from IPython.core.application import (
@@ -105,7 +105,7 b' class BaseParallelApplication(BaseIPythonApplication):'
     
     def _log_format_default(self):
         """override default log format to include time"""
-        return u"%(asctime)s.%(msecs).03d [%(name)s] %(message)s"
+        return u"%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s"
 
     work_dir = Unicode(os.getcwdu(), config=True,
         help='Set the working dir for the process.'
@@ -191,8 +191,8 b' class BaseParallelApplication(BaseIPythonApplication):'
         else:
             self._log_handler = self.log.handlers[0]
         # Add timestamps to log format:
-        self._log_formatter = logging.Formatter(self.log_format,
-                                                datefmt="%Y-%m-%d %H:%M:%S")
+        self._log_formatter = LevelFormatter(self.log_format,
+                                                datefmt=self.log_datefmt)
         self._log_handler.setFormatter(self._log_formatter)
         # do not propagate log messages to root logger
         # ipcluster app will sometimes print duplicate messages during shutdown

README.rst

@@ -47,3 +47,18 b' environment so that you can work on your local repo copy and run it from anywher'
 
 The same process applies for other parts, such as the qtconsole (the
 ``extras_require`` attribute in the setup.py file lists all the possibilities).
+
+Git Hooks and Submodules
+************************
+
+IPython now uses git submodules to ship its javascript dependencies.
+If you run IPython from git master, you may need to update submodules once in a while with::
+
+    $ git submodule update
+
+or::
+
+    $ python setup.py submodule
+
+We have some git hooks for helping keep your submodules always in sync,
+see our ``git-hooks`` directory for more info.

setup.py

@@ -68,7 +68,7 b' from setupbase import ('
     find_data_files,
     check_for_dependencies,
     git_prebuild,
-    check_for_submodules,
+    check_submodule_status,
     update_submodules,
     require_submodules,
     UpdateSubmodules,
@@ -112,21 +112,37 b" if os_name == 'windows' and 'sdist' in sys.argv:"
 #-------------------------------------------------------------------------------
 # Make sure we aren't trying to run without submodules
 #-------------------------------------------------------------------------------
+here = os.path.abspath(os.path.dirname(__file__))
 
-def ensure_submodules_exist():
-    """Check out git submodules before distutils can do anything
-    
-    Because distutils cannot be trusted to update the tree
-    after everything has been set in motion.
+def require_clean_submodules():
+    """Check on git submodules before distutils can do anything
+
+    Since distutils cannot be trusted to update the tree
+    after everything has been set in motion,
+    this is not a distutils command.
     """
     # don't do anything if nothing is actually supposed to happen
-    for do_nothing in ('-h', '--help', '--help-commands', 'clean'):
+    for do_nothing in ('-h', '--help', '--help-commands', 'clean', 'submodule'):
         if do_nothing in sys.argv:
             return
-    if not check_for_submodules():
-        update_submodules()
 
-ensure_submodules_exist()
+    status = check_submodule_status(here)
+
+    if status == "missing":
+        print("checking out submodules for the first time")
+        update_submodules(here)
+    elif status == "unclean":
+        print('\n'.join([
+            "Cannot build / install IPython with unclean submodules",
+            "Please update submodules with",
+            "    python setup.py submodule",
+            "or",
+            "    git submodule update",
+            "or commit any submodule changes you have made."
+        ]))
+        sys.exit(1)
+
+require_clean_submodules()
 
 #-------------------------------------------------------------------------------
 # Things related to the IPython documentation

setupbase.py

@@ -373,27 +373,10 b' def check_for_dependencies():'
 # VCS related
 #---------------------------------------------------------------------------
 
-def check_for_submodules():
-    """return False if there are any submodules that need to be checked out,
-    True otherwise.
-    
-    This doesn't check if they are up to date, only existence.
-    """
-    here = os.path.dirname(__file__)
-    submodules = [
-        os.path.join(here, 'IPython', 'frontend', 'html', 'notebook', 'static', 'components')
-    ]
-    for submodule in submodules:
-        if not os.path.exists(submodule):
-            return False
-    return True
+here = os.path.abspath(os.path.dirname(__file__))
 
-def update_submodules():
-    """update git submodules"""
-    import subprocess
-    print("updating git submodules")
-    subprocess.check_call('git submodule init'.split())
-    subprocess.check_call('git submodule update --recursive'.split())
+# utils.submodule has checks for submodule status
+execfile(pjoin('IPython','utils','submodule.py'), globals())
 
 class UpdateSubmodules(Command):
     """Update git submodules
@@ -418,12 +401,10 b' class UpdateSubmodules(Command):'
             failure = e
             print(e)
         
-        if not check_for_submodules():
+        if not check_submodule_status(here) == 'clean':
             print("submodules could not be checked out")
             sys.exit(1)
-        
-        # re-scan package data after update
-        self.distribution.package_data = find_package_data()
+
 
 def git_prebuild(pkg_dir, build_cmd=build_py):
     """Return extended build or sdist command class for recording commit
@@ -438,10 +419,6 b' def git_prebuild(pkg_dir, build_cmd=build_py):'
     class MyBuildPy(build_cmd):
         ''' Subclass to write commit data into installation tree '''
         def run(self):
-            if not check_for_submodules():
-                print("submodules missing! Run `setup.py submodule` and try again")
-                sys.exit(1)
-            
             build_cmd.run(self)
             # this one will only fire for build commands
             if hasattr(self, 'build_lib'):
@@ -478,14 +455,14 b' def git_prebuild(pkg_dir, build_cmd=build_py):'
                     '# GENERATED BY setup.py\n',
                     'commit = "%s"\n' % repo_commit,
                 ])
-    return MyBuildPy
+    return require_submodules(MyBuildPy)
 
 
 def require_submodules(command):
     """decorator for instructing a command to check for submodules before running"""
     class DecoratedCommand(command):
         def run(self):
-            if not check_for_submodules():
+            if not check_submodule_status(here) == 'clean':
                 print("submodules missing! Run `setup.py submodule` and try again")
                 sys.exit(1)
             command.run(self)