diff --git a/IPython/core/profileapp.py b/IPython/core/profileapp.py index 021b58a..b09b0aa 100644 --- a/IPython/core/profileapp.py +++ b/IPython/core/profileapp.py @@ -92,6 +92,29 @@ ipython profile list -h # show the help string for the list subcommand #----------------------------------------------------------------------------- +def list_profiles_in(path): + """list profiles in a given root directory""" + files = os.listdir(path) + profiles = [] + for f in files: + full_path = os.path.join(path, f) + if os.path.isdir(full_path) and f.startswith('profile_'): + profiles.append(f.split('_',1)[-1]) + return profiles + + +def list_bundled_profiles(): + """list profiles that are bundled with IPython.""" + path = os.path.join(get_ipython_package_dir(), u'config', u'profile') + files = os.listdir(path) + profiles = [] + for profile in files: + full_path = os.path.join(path, profile) + if os.path.isdir(full_path): + profiles.append(profile) + return profiles + + class ProfileList(Application): name = u'ipython-profile' description = list_help @@ -115,35 +138,15 @@ class ProfileList(Application): the environment variable IPYTHON_DIR. """ ) - - def _list_profiles_in(self, path): - """list profiles in a given root directory""" - files = os.listdir(path) - profiles = [] - for f in files: - full_path = os.path.join(path, f) - if os.path.isdir(full_path) and f.startswith('profile_'): - profiles.append(f.split('_',1)[-1]) - return profiles - - def _list_bundled_profiles(self): - """list profiles in a given root directory""" - path = os.path.join(get_ipython_package_dir(), u'config', u'profile') - files = os.listdir(path) - profiles = [] - for profile in files: - full_path = os.path.join(path, profile) - if os.path.isdir(full_path): - profiles.append(profile) - return profiles - + + def _print_profiles(self, profiles): """print list of profiles, indented.""" for profile in profiles: print ' %s' % profile - + def list_profile_dirs(self): - profiles = self._list_bundled_profiles() + profiles = list_bundled_profiles() if profiles: print print "Available profiles in IPython:" @@ -153,13 +156,13 @@ class ProfileList(Application): print " into your IPython directory (%s)," % self.ipython_dir print " where you can customize it." - profiles = self._list_profiles_in(self.ipython_dir) + profiles = list_profiles_in(self.ipython_dir) if profiles: print print "Available profiles in %s:" % self.ipython_dir self._print_profiles(profiles) - profiles = self._list_profiles_in(os.getcwdu()) + profiles = list_profiles_in(os.getcwdu()) if profiles: print print "Available profiles in current directory (%s):" % os.getcwdu() diff --git a/IPython/frontend/html/notebook/clustermanager.py b/IPython/frontend/html/notebook/clustermanager.py new file mode 100644 index 0000000..4fc8626 --- /dev/null +++ b/IPython/frontend/html/notebook/clustermanager.py @@ -0,0 +1,90 @@ +"""Manage IPython.parallel clusters in the notebook. + +Authors: + +* Brian Granger +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2008-2011 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import datetime +import os +import uuid +import glob + +from tornado import web +from zmq.eventloop import ioloop + +from IPython.config.configurable import LoggingConfigurable +from IPython.utils.traitlets import Unicode, List, Dict, Bool +from IPython.parallel.apps.launcher import IPClusterLauncher +from IPython.core.profileapp import list_profiles_in, list_bundled_profiles +from IPython.utils.path import get_ipython_dir, get_ipython_package_dir + +#----------------------------------------------------------------------------- +# Classes +#----------------------------------------------------------------------------- + +class ClusterManager(LoggingConfigurable): + + profiles = Dict() + + + def list_profile_names(self): + """List all profiles in the ipython_dir and cwd. + """ + profiles = list_profiles_in(get_ipython_dir()) + profiles += list_profiles_in(os.getcwdu()) + return profiles + + + def list_profiles(self): + profiles = self.list_profile_names() + result = [self.profile_info(p) for p in profiles] + return result + + + def profile_info(self, profile): + if profile not in self.list_profile_names(): + raise web.HTTPError(404, u'profile not found') + result = dict(profile=profile) + data = self.profiles.get(profile) + if data is None: + result['status'] = 'stopped' + else: + result['status'] = 'running' + result['n'] = data['n'] + return result + + def start_cluster(self, profile, n=4): + """Start a cluster for a given profile.""" + if profile not in self.list_profile_names(): + raise web.HTTPError(404, u'profile not found') + if profile in self.profiles: + raise web.HTTPError(409, u'cluster already running') + launcher = IPClusterLauncher(ipcluster_profile=profile, ipcluster_n=n) + launcher.start() + self.profiles[profile] = { + 'launcher': launcher, + 'n': n + } + + def stop_cluster(self, profile): + """Stop a cluster for a given profile.""" + if profile not in self.profiles: + raise web.HTTPError(409, u'cluster not running') + launcher = self.profiles.pop(profile)['launcher'] + launcher.stop() + + def stop_all_clusters(self): + for p in self.profiles.values(): + p['launcher'].stop() diff --git a/IPython/frontend/html/notebook/handlers.py b/IPython/frontend/html/notebook/handlers.py index faba77d..ba1e5a2 100644 --- a/IPython/frontend/html/notebook/handlers.py +++ b/IPython/frontend/html/notebook/handlers.py @@ -587,7 +587,6 @@ class NotebookRootHandler(AuthenticatedHandler): @authenticate_unless_readonly def get(self): - nbm = self.application.notebook_manager files = nbm.list_notebooks() self.finish(jsonapi.dumps(files)) @@ -661,6 +660,41 @@ class NotebookCopyHandler(AuthenticatedHandler): mathjax_url=self.application.ipython_app.mathjax_url, ) + +#----------------------------------------------------------------------------- +# Cluster handlers +#----------------------------------------------------------------------------- + + +class MainClusterHandler(AuthenticatedHandler): + + @web.authenticated + def get(self): + cm = self.application.cluster_manager + self.finish(jsonapi.dumps(cm.list_profiles())) + + +class ClusterProfileHandler(AuthenticatedHandler): + + @web.authenticated + def get(self, profile): + cm = self.application.cluster_manager + self.finish(jsonapi.dumps(cm.profile_info(profile))) + + +class ClusterActionHandler(AuthenticatedHandler): + + @web.authenticated + def post(self, profile, action): + cm = self.application.cluster_manager + if action == 'start': + n = int(self.get_argument('n', default=4)) + cm.start_cluster(profile, n) + if action == 'stop': + cm.stop_cluster(profile) + self.finish() + + #----------------------------------------------------------------------------- # RST web service handlers #----------------------------------------------------------------------------- diff --git a/IPython/frontend/html/notebook/notebookapp.py b/IPython/frontend/html/notebook/notebookapp.py index 09b91de..0c5be71 100644 --- a/IPython/frontend/html/notebook/notebookapp.py +++ b/IPython/frontend/html/notebook/notebookapp.py @@ -49,9 +49,11 @@ from .handlers import (LoginHandler, LogoutHandler, ProjectDashboardHandler, NewHandler, NamedNotebookHandler, MainKernelHandler, KernelHandler, KernelActionHandler, IOPubHandler, ShellHandler, NotebookRootHandler, NotebookHandler, NotebookCopyHandler, - RSTHandler, AuthenticatedFileHandler, PrintNotebookHandler + RSTHandler, AuthenticatedFileHandler, PrintNotebookHandler, + MainClusterHandler, ClusterProfileHandler, ClusterActionHandler ) from .notebookmanager import NotebookManager +from .clustermanager import ClusterManager from IPython.config.application import catch_config_error, boolean_flag from IPython.core.application import BaseIPythonApplication @@ -74,6 +76,9 @@ from IPython.utils import py3compat _kernel_id_regex = r"(?P\w+-\w+-\w+-\w+-\w+)" _kernel_action_regex = r"(?Prestart|interrupt)" _notebook_id_regex = r"(?P\w+-\w+-\w+-\w+-\w+)" +_profile_regex = r"(?P[a-zA-Z0-9]+)" +_cluster_action_regex = r"(?Pstart|stop)" + LOCALHOST = '127.0.0.1' @@ -101,7 +106,8 @@ def url_path_join(a,b): class NotebookWebApplication(web.Application): - def __init__(self, ipython_app, kernel_manager, notebook_manager, log, + def __init__(self, ipython_app, kernel_manager, notebook_manager, + cluster_manager, log, base_project_url, settings_overrides): handlers = [ (r"/", ProjectDashboardHandler), @@ -120,6 +126,9 @@ class NotebookWebApplication(web.Application): (r"/notebooks/%s" % _notebook_id_regex, NotebookHandler), (r"/rstservice/render", RSTHandler), (r"/files/(.*)", AuthenticatedFileHandler, {'path' : notebook_manager.notebook_dir}), + (r"/clusters", MainClusterHandler), + (r"/clusters/%s/%s" % (_profile_regex, _cluster_action_regex), ClusterActionHandler), + (r"/clusters/%s" % _profile_regex, ClusterProfileHandler), ] settings = dict( template_path=os.path.join(os.path.dirname(__file__), "templates"), @@ -151,10 +160,11 @@ class NotebookWebApplication(web.Application): super(NotebookWebApplication, self).__init__(new_handlers, **settings) self.kernel_manager = kernel_manager - self.log = log self.notebook_manager = notebook_manager + self.cluster_manager = cluster_manager self.ipython_app = ipython_app self.read_only = self.ipython_app.read_only + self.log = log #----------------------------------------------------------------------------- @@ -395,6 +405,7 @@ class NotebookApp(BaseIPythonApplication): ) self.notebook_manager = NotebookManager(config=self.config, log=self.log) self.notebook_manager.list_notebooks() + self.cluster_manager = ClusterManager(config=self.config, log=self.log) def init_logging(self): super(NotebookApp, self).init_logging() @@ -406,7 +417,8 @@ class NotebookApp(BaseIPythonApplication): def init_webapp(self): """initialize tornado webapp and httpserver""" self.web_app = NotebookWebApplication( - self, self.kernel_manager, self.notebook_manager, self.log, + self, self.kernel_manager, self.notebook_manager, + self.cluster_manager, self.log, self.base_project_url, self.webapp_settings ) if self.certfile: diff --git a/IPython/parallel/apps/launcher.py b/IPython/parallel/apps/launcher.py index 8095473..08f8d49 100644 --- a/IPython/parallel/apps/launcher.py +++ b/IPython/parallel/apps/launcher.py @@ -1164,14 +1164,16 @@ class IPClusterLauncher(LocalProcessLauncher): ipcluster_cmd = List(ipcluster_cmd_argv, config=True, help="Popen command for ipcluster") ipcluster_args = List( - ['--clean-logs', '--log-to-file', '--log-level=%i'%logging.INFO], config=True, + ['--clean-logs=True', '--log-to-file', '--log-level=%i'%logging.INFO], config=True, help="Command line arguments to pass to ipcluster.") ipcluster_subcommand = Unicode('start') + ipcluster_profile = Unicode('default') ipcluster_n = Integer(2) def find_args(self): return self.ipcluster_cmd + [self.ipcluster_subcommand] + \ - ['--n=%i'%self.ipcluster_n] + self.ipcluster_args + ['--n=%i'%self.ipcluster_n, '--profile=%s'%self.ipcluster_profile] + \ + self.ipcluster_args def start(self): return super(IPClusterLauncher, self).start()