From 014c18a79c62b89a508e064cee640bfe71fe7bf8 2012-03-08 19:27:29
From: Brian Granger <ellisonbg@gmail.com>
Date: 2012-03-08 19:27:29
Subject: [PATCH] First version of cluster web service.
This exposes ipcluster's over the web. The current implementation
uses IPClusterLauncher to run ipcluster in a separate process.
Here is the URL scheme we are using:
GET /clusters => list available clusters
GET /cluster/profile => list info for cluster with profile
POST /cluster/profile/start => start a cluster
POST /cluster/profile/stop => stop a cluster
---
diff --git a/IPython/core/profileapp.py b/IPython/core/profileapp.py
index 021b58a..b09b0aa 100644
--- a/IPython/core/profileapp.py
+++ b/IPython/core/profileapp.py
@@ -92,6 +92,29 @@ ipython profile list -h # show the help string for the list subcommand
#-----------------------------------------------------------------------------
+def list_profiles_in(path):
+ """list profiles in a given root directory"""
+ files = os.listdir(path)
+ profiles = []
+ for f in files:
+ full_path = os.path.join(path, f)
+ if os.path.isdir(full_path) and f.startswith('profile_'):
+ profiles.append(f.split('_',1)[-1])
+ return profiles
+
+
+def list_bundled_profiles():
+ """list profiles that are bundled with IPython."""
+ path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
+ files = os.listdir(path)
+ profiles = []
+ for profile in files:
+ full_path = os.path.join(path, profile)
+ if os.path.isdir(full_path):
+ profiles.append(profile)
+ return profiles
+
+
class ProfileList(Application):
name = u'ipython-profile'
description = list_help
@@ -115,35 +138,15 @@ class ProfileList(Application):
the environment variable IPYTHON_DIR.
"""
)
-
- def _list_profiles_in(self, path):
- """list profiles in a given root directory"""
- files = os.listdir(path)
- profiles = []
- for f in files:
- full_path = os.path.join(path, f)
- if os.path.isdir(full_path) and f.startswith('profile_'):
- profiles.append(f.split('_',1)[-1])
- return profiles
-
- def _list_bundled_profiles(self):
- """list profiles in a given root directory"""
- path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
- files = os.listdir(path)
- profiles = []
- for profile in files:
- full_path = os.path.join(path, profile)
- if os.path.isdir(full_path):
- profiles.append(profile)
- return profiles
-
+
+
def _print_profiles(self, profiles):
"""print list of profiles, indented."""
for profile in profiles:
print ' %s' % profile
-
+
def list_profile_dirs(self):
- profiles = self._list_bundled_profiles()
+ profiles = list_bundled_profiles()
if profiles:
print
print "Available profiles in IPython:"
@@ -153,13 +156,13 @@ class ProfileList(Application):
print " into your IPython directory (%s)," % self.ipython_dir
print " where you can customize it."
- profiles = self._list_profiles_in(self.ipython_dir)
+ profiles = list_profiles_in(self.ipython_dir)
if profiles:
print
print "Available profiles in %s:" % self.ipython_dir
self._print_profiles(profiles)
- profiles = self._list_profiles_in(os.getcwdu())
+ profiles = list_profiles_in(os.getcwdu())
if profiles:
print
print "Available profiles in current directory (%s):" % os.getcwdu()
diff --git a/IPython/frontend/html/notebook/clustermanager.py b/IPython/frontend/html/notebook/clustermanager.py
new file mode 100644
index 0000000..4fc8626
--- /dev/null
+++ b/IPython/frontend/html/notebook/clustermanager.py
@@ -0,0 +1,90 @@
+"""Manage IPython.parallel clusters in the notebook.
+
+Authors:
+
+* Brian Granger
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (C) 2008-2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import datetime
+import os
+import uuid
+import glob
+
+from tornado import web
+from zmq.eventloop import ioloop
+
+from IPython.config.configurable import LoggingConfigurable
+from IPython.utils.traitlets import Unicode, List, Dict, Bool
+from IPython.parallel.apps.launcher import IPClusterLauncher
+from IPython.core.profileapp import list_profiles_in, list_bundled_profiles
+from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+class ClusterManager(LoggingConfigurable):
+
+ profiles = Dict()
+
+
+ def list_profile_names(self):
+ """List all profiles in the ipython_dir and cwd.
+ """
+ profiles = list_profiles_in(get_ipython_dir())
+ profiles += list_profiles_in(os.getcwdu())
+ return profiles
+
+
+ def list_profiles(self):
+ profiles = self.list_profile_names()
+ result = [self.profile_info(p) for p in profiles]
+ return result
+
+
+ def profile_info(self, profile):
+ if profile not in self.list_profile_names():
+ raise web.HTTPError(404, u'profile not found')
+ result = dict(profile=profile)
+ data = self.profiles.get(profile)
+ if data is None:
+ result['status'] = 'stopped'
+ else:
+ result['status'] = 'running'
+ result['n'] = data['n']
+ return result
+
+ def start_cluster(self, profile, n=4):
+ """Start a cluster for a given profile."""
+ if profile not in self.list_profile_names():
+ raise web.HTTPError(404, u'profile not found')
+ if profile in self.profiles:
+ raise web.HTTPError(409, u'cluster already running')
+ launcher = IPClusterLauncher(ipcluster_profile=profile, ipcluster_n=n)
+ launcher.start()
+ self.profiles[profile] = {
+ 'launcher': launcher,
+ 'n': n
+ }
+
+ def stop_cluster(self, profile):
+ """Stop a cluster for a given profile."""
+ if profile not in self.profiles:
+ raise web.HTTPError(409, u'cluster not running')
+ launcher = self.profiles.pop(profile)['launcher']
+ launcher.stop()
+
+ def stop_all_clusters(self):
+ for p in self.profiles.values():
+ p['launcher'].stop()
diff --git a/IPython/frontend/html/notebook/handlers.py b/IPython/frontend/html/notebook/handlers.py
index faba77d..ba1e5a2 100644
--- a/IPython/frontend/html/notebook/handlers.py
+++ b/IPython/frontend/html/notebook/handlers.py
@@ -587,7 +587,6 @@ class NotebookRootHandler(AuthenticatedHandler):
@authenticate_unless_readonly
def get(self):
-
nbm = self.application.notebook_manager
files = nbm.list_notebooks()
self.finish(jsonapi.dumps(files))
@@ -661,6 +660,41 @@ class NotebookCopyHandler(AuthenticatedHandler):
mathjax_url=self.application.ipython_app.mathjax_url,
)
+
+#-----------------------------------------------------------------------------
+# Cluster handlers
+#-----------------------------------------------------------------------------
+
+
+class MainClusterHandler(AuthenticatedHandler):
+
+ @web.authenticated
+ def get(self):
+ cm = self.application.cluster_manager
+ self.finish(jsonapi.dumps(cm.list_profiles()))
+
+
+class ClusterProfileHandler(AuthenticatedHandler):
+
+ @web.authenticated
+ def get(self, profile):
+ cm = self.application.cluster_manager
+ self.finish(jsonapi.dumps(cm.profile_info(profile)))
+
+
+class ClusterActionHandler(AuthenticatedHandler):
+
+ @web.authenticated
+ def post(self, profile, action):
+ cm = self.application.cluster_manager
+ if action == 'start':
+ n = int(self.get_argument('n', default=4))
+ cm.start_cluster(profile, n)
+ if action == 'stop':
+ cm.stop_cluster(profile)
+ self.finish()
+
+
#-----------------------------------------------------------------------------
# RST web service handlers
#-----------------------------------------------------------------------------
diff --git a/IPython/frontend/html/notebook/notebookapp.py b/IPython/frontend/html/notebook/notebookapp.py
index 09b91de..0c5be71 100644
--- a/IPython/frontend/html/notebook/notebookapp.py
+++ b/IPython/frontend/html/notebook/notebookapp.py
@@ -49,9 +49,11 @@ from .handlers import (LoginHandler, LogoutHandler,
ProjectDashboardHandler, NewHandler, NamedNotebookHandler,
MainKernelHandler, KernelHandler, KernelActionHandler, IOPubHandler,
ShellHandler, NotebookRootHandler, NotebookHandler, NotebookCopyHandler,
- RSTHandler, AuthenticatedFileHandler, PrintNotebookHandler
+ RSTHandler, AuthenticatedFileHandler, PrintNotebookHandler,
+ MainClusterHandler, ClusterProfileHandler, ClusterActionHandler
)
from .notebookmanager import NotebookManager
+from .clustermanager import ClusterManager
from IPython.config.application import catch_config_error, boolean_flag
from IPython.core.application import BaseIPythonApplication
@@ -74,6 +76,9 @@ from IPython.utils import py3compat
_kernel_id_regex = r"(?P<kernel_id>\w+-\w+-\w+-\w+-\w+)"
_kernel_action_regex = r"(?P<action>restart|interrupt)"
_notebook_id_regex = r"(?P<notebook_id>\w+-\w+-\w+-\w+-\w+)"
+_profile_regex = r"(?P<profile>[a-zA-Z0-9]+)"
+_cluster_action_regex = r"(?P<action>start|stop)"
+
LOCALHOST = '127.0.0.1'
@@ -101,7 +106,8 @@ def url_path_join(a,b):
class NotebookWebApplication(web.Application):
- def __init__(self, ipython_app, kernel_manager, notebook_manager, log,
+ def __init__(self, ipython_app, kernel_manager, notebook_manager,
+ cluster_manager, log,
base_project_url, settings_overrides):
handlers = [
(r"/", ProjectDashboardHandler),
@@ -120,6 +126,9 @@ class NotebookWebApplication(web.Application):
(r"/notebooks/%s" % _notebook_id_regex, NotebookHandler),
(r"/rstservice/render", RSTHandler),
(r"/files/(.*)", AuthenticatedFileHandler, {'path' : notebook_manager.notebook_dir}),
+ (r"/clusters", MainClusterHandler),
+ (r"/clusters/%s/%s" % (_profile_regex, _cluster_action_regex), ClusterActionHandler),
+ (r"/clusters/%s" % _profile_regex, ClusterProfileHandler),
]
settings = dict(
template_path=os.path.join(os.path.dirname(__file__), "templates"),
@@ -151,10 +160,11 @@ class NotebookWebApplication(web.Application):
super(NotebookWebApplication, self).__init__(new_handlers, **settings)
self.kernel_manager = kernel_manager
- self.log = log
self.notebook_manager = notebook_manager
+ self.cluster_manager = cluster_manager
self.ipython_app = ipython_app
self.read_only = self.ipython_app.read_only
+ self.log = log
#-----------------------------------------------------------------------------
@@ -395,6 +405,7 @@ class NotebookApp(BaseIPythonApplication):
)
self.notebook_manager = NotebookManager(config=self.config, log=self.log)
self.notebook_manager.list_notebooks()
+ self.cluster_manager = ClusterManager(config=self.config, log=self.log)
def init_logging(self):
super(NotebookApp, self).init_logging()
@@ -406,7 +417,8 @@ class NotebookApp(BaseIPythonApplication):
def init_webapp(self):
"""initialize tornado webapp and httpserver"""
self.web_app = NotebookWebApplication(
- self, self.kernel_manager, self.notebook_manager, self.log,
+ self, self.kernel_manager, self.notebook_manager,
+ self.cluster_manager, self.log,
self.base_project_url, self.webapp_settings
)
if self.certfile:
diff --git a/IPython/parallel/apps/launcher.py b/IPython/parallel/apps/launcher.py
index 8095473..08f8d49 100644
--- a/IPython/parallel/apps/launcher.py
+++ b/IPython/parallel/apps/launcher.py
@@ -1164,14 +1164,16 @@ class IPClusterLauncher(LocalProcessLauncher):
ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
help="Popen command for ipcluster")
ipcluster_args = List(
- ['--clean-logs', '--log-to-file', '--log-level=%i'%logging.INFO], config=True,
+ ['--clean-logs=True', '--log-to-file', '--log-level=%i'%logging.INFO], config=True,
help="Command line arguments to pass to ipcluster.")
ipcluster_subcommand = Unicode('start')
+ ipcluster_profile = Unicode('default')
ipcluster_n = Integer(2)
def find_args(self):
return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
- ['--n=%i'%self.ipcluster_n] + self.ipcluster_args
+ ['--n=%i'%self.ipcluster_n, '--profile=%s'%self.ipcluster_profile] + \
+ self.ipcluster_args
def start(self):
return super(IPClusterLauncher, self).start()