First version of cluster web service.

This exposes ipcluster's over the web. The current implementation
uses IPClusterLauncher to run ipcluster in a separate process.
Here is the URL scheme we are using:

GET /clusters => list available clusters
GET /cluster/profile => list info for cluster with profile
POST /cluster/profile/start => start a cluster
POST /cluster/profile/stop => stop a cluster

IPython/frontend/html/notebook/clustermanager.py

@@ -0,0 +1,90 b''
+"""Manage IPython.parallel clusters in the notebook.
+
+Authors:
+
+* Brian Granger
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2008-2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import datetime
+import os
+import uuid
+import glob
+
+from tornado import web
+from zmq.eventloop import ioloop
+
+from IPython.config.configurable import LoggingConfigurable
+from IPython.utils.traitlets import Unicode, List, Dict, Bool
+from IPython.parallel.apps.launcher import IPClusterLauncher
+from IPython.core.profileapp import list_profiles_in, list_bundled_profiles
+from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+class ClusterManager(LoggingConfigurable):
+
+    profiles = Dict()
+
+
+    def list_profile_names(self):
+        """List all profiles in the ipython_dir and cwd.
+        """
+        profiles = list_profiles_in(get_ipython_dir())
+        profiles += list_profiles_in(os.getcwdu())
+        return profiles
+
+
+    def list_profiles(self):
+        profiles = self.list_profile_names()
+        result = [self.profile_info(p) for p in profiles]
+        return result
+
+
+    def profile_info(self, profile):
+        if profile not in self.list_profile_names():
+            raise web.HTTPError(404, u'profile not found')
+        result = dict(profile=profile)
+        data = self.profiles.get(profile)
+        if data is None:
+            result['status'] = 'stopped'
+        else:
+            result['status'] = 'running'
+            result['n'] = data['n']
+        return result
+
+    def start_cluster(self, profile, n=4):
+        """Start a cluster for a given profile."""
+        if profile not in self.list_profile_names():
+            raise web.HTTPError(404, u'profile not found')
+        if profile in self.profiles:
+            raise web.HTTPError(409, u'cluster already running')
+        launcher = IPClusterLauncher(ipcluster_profile=profile, ipcluster_n=n)
+        launcher.start()
+        self.profiles[profile] = {
+            'launcher': launcher,
+            'n': n
+        }
+
+    def stop_cluster(self, profile):
+        """Stop a cluster for a given profile."""
+        if profile not in self.profiles:
+            raise web.HTTPError(409, u'cluster not running')
+        launcher = self.profiles.pop(profile)['launcher']
+        launcher.stop()
+
+    def stop_all_clusters(self):
+        for p in self.profiles.values():
+            p['launcher'].stop()

IPython/core/profileapp.py

@@ -92,6 +92,29 b' ipython profile list -h # show the help string for the list subcommand'
 #-----------------------------------------------------------------------------
 
 
+def list_profiles_in(path):
+    """list profiles in a given root directory"""
+    files = os.listdir(path)
+    profiles = []
+    for f in files:
+        full_path = os.path.join(path, f)
+        if os.path.isdir(full_path) and f.startswith('profile_'):
+            profiles.append(f.split('_',1)[-1])
+    return profiles
+
+
+def list_bundled_profiles():
+    """list profiles that are bundled with IPython."""
+    path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
+    files = os.listdir(path)
+    profiles = []
+    for profile in files:
+        full_path = os.path.join(path, profile)
+        if os.path.isdir(full_path):
+            profiles.append(profile)
+    return profiles
+
+
 class ProfileList(Application):
     name = u'ipython-profile'
     description = list_help
@@ -115,35 +138,15 b' class ProfileList(Application):'
         the environment variable IPYTHON_DIR.
         """
     )
-    
-    def _list_profiles_in(self, path):
-        """list profiles in a given root directory"""
-        files = os.listdir(path)
-        profiles = []
-        for f in files:
-            full_path = os.path.join(path, f)
-            if os.path.isdir(full_path) and f.startswith('profile_'):
-                profiles.append(f.split('_',1)[-1])
-        return profiles
-    
-    def _list_bundled_profiles(self):
-        """list profiles in a given root directory"""
-        path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
-        files = os.listdir(path)
-        profiles = []
-        for profile in files:
-            full_path = os.path.join(path, profile)
-            if os.path.isdir(full_path):
-                profiles.append(profile)
-        return profiles
-    
+
+
     def _print_profiles(self, profiles):
         """print list of profiles, indented."""
         for profile in profiles:
             print '    %s' % profile
-    
+
     def list_profile_dirs(self):
-        profiles = self._list_bundled_profiles()
+        profiles = list_bundled_profiles()
         if profiles:
             print
             print "Available profiles in IPython:"
@@ -153,13 +156,13 b' class ProfileList(Application):'
             print "    into your IPython directory (%s)," % self.ipython_dir
             print "    where you can customize it."
         
-        profiles = self._list_profiles_in(self.ipython_dir)
+        profiles = list_profiles_in(self.ipython_dir)
         if profiles:
             print
             print "Available profiles in %s:" % self.ipython_dir
             self._print_profiles(profiles)
         
-        profiles = self._list_profiles_in(os.getcwdu())
+        profiles = list_profiles_in(os.getcwdu())
         if profiles:
             print
             print "Available profiles in current directory (%s):" % os.getcwdu()

IPython/frontend/html/notebook/handlers.py

@@ -587,7 +587,6 b' class NotebookRootHandler(AuthenticatedHandler):'
 
     @authenticate_unless_readonly
     def get(self):
-        
         nbm = self.application.notebook_manager
         files = nbm.list_notebooks()
         self.finish(jsonapi.dumps(files))
@@ -661,6 +660,41 b' class NotebookCopyHandler(AuthenticatedHandler):'
             mathjax_url=self.application.ipython_app.mathjax_url,
         )
 
+
+#-----------------------------------------------------------------------------
+# Cluster handlers
+#-----------------------------------------------------------------------------
+
+
+class MainClusterHandler(AuthenticatedHandler):
+
+    @web.authenticated
+    def get(self):
+        cm = self.application.cluster_manager
+        self.finish(jsonapi.dumps(cm.list_profiles()))
+
+
+class ClusterProfileHandler(AuthenticatedHandler):
+
+    @web.authenticated
+    def get(self, profile):
+        cm = self.application.cluster_manager
+        self.finish(jsonapi.dumps(cm.profile_info(profile)))
+
+
+class ClusterActionHandler(AuthenticatedHandler):
+
+    @web.authenticated
+    def post(self, profile, action):
+        cm = self.application.cluster_manager
+        if action == 'start':
+            n = int(self.get_argument('n', default=4))
+            cm.start_cluster(profile, n)
+        if action == 'stop':
+            cm.stop_cluster(profile)
+        self.finish()
+
+
 #-----------------------------------------------------------------------------
 # RST web service handlers
 #-----------------------------------------------------------------------------

IPython/frontend/html/notebook/notebookapp.py

@@ -49,9 +49,11 b' from .handlers import (LoginHandler, LogoutHandler,'
     ProjectDashboardHandler, NewHandler, NamedNotebookHandler,
     MainKernelHandler, KernelHandler, KernelActionHandler, IOPubHandler,
     ShellHandler, NotebookRootHandler, NotebookHandler, NotebookCopyHandler,
-    RSTHandler, AuthenticatedFileHandler, PrintNotebookHandler
+    RSTHandler, AuthenticatedFileHandler, PrintNotebookHandler,
+    MainClusterHandler, ClusterProfileHandler, ClusterActionHandler
 )
 from .notebookmanager import NotebookManager
+from .clustermanager import ClusterManager
 
 from IPython.config.application import catch_config_error, boolean_flag
 from IPython.core.application import BaseIPythonApplication
@@ -74,6 +76,9 b' from IPython.utils import py3compat'
 _kernel_id_regex = r"(?P<kernel_id>\w+-\w+-\w+-\w+-\w+)"
 _kernel_action_regex = r"(?P<action>restart|interrupt)"
 _notebook_id_regex = r"(?P<notebook_id>\w+-\w+-\w+-\w+-\w+)"
+_profile_regex = r"(?P<profile>[a-zA-Z0-9]+)"
+_cluster_action_regex = r"(?P<action>start|stop)"
+
 
 LOCALHOST = '127.0.0.1'
 
@@ -101,7 +106,8 b' def url_path_join(a,b):'
 
 class NotebookWebApplication(web.Application):
 
-    def __init__(self, ipython_app, kernel_manager, notebook_manager, log,
+    def __init__(self, ipython_app, kernel_manager, notebook_manager, 
+                 cluster_manager, log,
                  base_project_url, settings_overrides):
         handlers = [
             (r"/", ProjectDashboardHandler),
@@ -120,6 +126,9 b' class NotebookWebApplication(web.Application):'
             (r"/notebooks/%s" % _notebook_id_regex, NotebookHandler),
             (r"/rstservice/render", RSTHandler),
             (r"/files/(.*)", AuthenticatedFileHandler, {'path' : notebook_manager.notebook_dir}),
+            (r"/clusters", MainClusterHandler),
+            (r"/clusters/%s/%s" % (_profile_regex, _cluster_action_regex), ClusterActionHandler),
+            (r"/clusters/%s" % _profile_regex, ClusterProfileHandler),
         ]
         settings = dict(
             template_path=os.path.join(os.path.dirname(__file__), "templates"),
@@ -151,10 +160,11 b' class NotebookWebApplication(web.Application):'
         super(NotebookWebApplication, self).__init__(new_handlers, **settings)
 
         self.kernel_manager = kernel_manager
-        self.log = log
         self.notebook_manager = notebook_manager
+        self.cluster_manager = cluster_manager
         self.ipython_app = ipython_app
         self.read_only = self.ipython_app.read_only
+        self.log = log
 
 
 #-----------------------------------------------------------------------------
@@ -395,6 +405,7 b' class NotebookApp(BaseIPythonApplication):'
         )
         self.notebook_manager = NotebookManager(config=self.config, log=self.log)
         self.notebook_manager.list_notebooks()
+        self.cluster_manager = ClusterManager(config=self.config, log=self.log)
 
     def init_logging(self):
         super(NotebookApp, self).init_logging()
@@ -406,7 +417,8 b' class NotebookApp(BaseIPythonApplication):'
     def init_webapp(self):
         """initialize tornado webapp and httpserver"""
         self.web_app = NotebookWebApplication(
-            self, self.kernel_manager, self.notebook_manager, self.log,
+            self, self.kernel_manager, self.notebook_manager, 
+            self.cluster_manager, self.log,
             self.base_project_url, self.webapp_settings
         )
         if self.certfile:

IPython/parallel/apps/launcher.py

@@ -1164,14 +1164,16 b' class IPClusterLauncher(LocalProcessLauncher):'
     ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
         help="Popen command for ipcluster")
     ipcluster_args = List(
-        ['--clean-logs', '--log-to-file', '--log-level=%i'%logging.INFO], config=True,
+        ['--clean-logs=True', '--log-to-file', '--log-level=%i'%logging.INFO], config=True,
         help="Command line arguments to pass to ipcluster.")
     ipcluster_subcommand = Unicode('start')
+    ipcluster_profile = Unicode('default')
     ipcluster_n = Integer(2)
 
     def find_args(self):
         return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
-            ['--n=%i'%self.ipcluster_n] + self.ipcluster_args
+            ['--n=%i'%self.ipcluster_n, '--profile=%s'%self.ipcluster_profile] + \
+            self.ipcluster_args
 
     def start(self):
         return super(IPClusterLauncher, self).start()