##// END OF EJS Templates
First version of cluster web service....
Brian Granger -
Show More
@@ -0,0 +1,90 b''
1 """Manage IPython.parallel clusters in the notebook.
2
3 Authors:
4
5 * Brian Granger
6 """
7
8 #-----------------------------------------------------------------------------
9 # Copyright (C) 2008-2011 The IPython Development Team
10 #
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
13 #-----------------------------------------------------------------------------
14
15 #-----------------------------------------------------------------------------
16 # Imports
17 #-----------------------------------------------------------------------------
18
19 import datetime
20 import os
21 import uuid
22 import glob
23
24 from tornado import web
25 from zmq.eventloop import ioloop
26
27 from IPython.config.configurable import LoggingConfigurable
28 from IPython.utils.traitlets import Unicode, List, Dict, Bool
29 from IPython.parallel.apps.launcher import IPClusterLauncher
30 from IPython.core.profileapp import list_profiles_in, list_bundled_profiles
31 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
32
33 #-----------------------------------------------------------------------------
34 # Classes
35 #-----------------------------------------------------------------------------
36
37 class ClusterManager(LoggingConfigurable):
38
39 profiles = Dict()
40
41
42 def list_profile_names(self):
43 """List all profiles in the ipython_dir and cwd.
44 """
45 profiles = list_profiles_in(get_ipython_dir())
46 profiles += list_profiles_in(os.getcwdu())
47 return profiles
48
49
50 def list_profiles(self):
51 profiles = self.list_profile_names()
52 result = [self.profile_info(p) for p in profiles]
53 return result
54
55
56 def profile_info(self, profile):
57 if profile not in self.list_profile_names():
58 raise web.HTTPError(404, u'profile not found')
59 result = dict(profile=profile)
60 data = self.profiles.get(profile)
61 if data is None:
62 result['status'] = 'stopped'
63 else:
64 result['status'] = 'running'
65 result['n'] = data['n']
66 return result
67
68 def start_cluster(self, profile, n=4):
69 """Start a cluster for a given profile."""
70 if profile not in self.list_profile_names():
71 raise web.HTTPError(404, u'profile not found')
72 if profile in self.profiles:
73 raise web.HTTPError(409, u'cluster already running')
74 launcher = IPClusterLauncher(ipcluster_profile=profile, ipcluster_n=n)
75 launcher.start()
76 self.profiles[profile] = {
77 'launcher': launcher,
78 'n': n
79 }
80
81 def stop_cluster(self, profile):
82 """Stop a cluster for a given profile."""
83 if profile not in self.profiles:
84 raise web.HTTPError(409, u'cluster not running')
85 launcher = self.profiles.pop(profile)['launcher']
86 launcher.stop()
87
88 def stop_all_clusters(self):
89 for p in self.profiles.values():
90 p['launcher'].stop()
@@ -92,6 +92,29 b' ipython profile list -h # show the help string for the list subcommand'
92 #-----------------------------------------------------------------------------
92 #-----------------------------------------------------------------------------
93
93
94
94
95 def list_profiles_in(path):
96 """list profiles in a given root directory"""
97 files = os.listdir(path)
98 profiles = []
99 for f in files:
100 full_path = os.path.join(path, f)
101 if os.path.isdir(full_path) and f.startswith('profile_'):
102 profiles.append(f.split('_',1)[-1])
103 return profiles
104
105
106 def list_bundled_profiles():
107 """list profiles that are bundled with IPython."""
108 path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
109 files = os.listdir(path)
110 profiles = []
111 for profile in files:
112 full_path = os.path.join(path, profile)
113 if os.path.isdir(full_path):
114 profiles.append(profile)
115 return profiles
116
117
95 class ProfileList(Application):
118 class ProfileList(Application):
96 name = u'ipython-profile'
119 name = u'ipython-profile'
97 description = list_help
120 description = list_help
@@ -116,26 +139,6 b' class ProfileList(Application):'
116 """
139 """
117 )
140 )
118
141
119 def _list_profiles_in(self, path):
120 """list profiles in a given root directory"""
121 files = os.listdir(path)
122 profiles = []
123 for f in files:
124 full_path = os.path.join(path, f)
125 if os.path.isdir(full_path) and f.startswith('profile_'):
126 profiles.append(f.split('_',1)[-1])
127 return profiles
128
129 def _list_bundled_profiles(self):
130 """list profiles in a given root directory"""
131 path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
132 files = os.listdir(path)
133 profiles = []
134 for profile in files:
135 full_path = os.path.join(path, profile)
136 if os.path.isdir(full_path):
137 profiles.append(profile)
138 return profiles
139
142
140 def _print_profiles(self, profiles):
143 def _print_profiles(self, profiles):
141 """print list of profiles, indented."""
144 """print list of profiles, indented."""
@@ -143,7 +146,7 b' class ProfileList(Application):'
143 print ' %s' % profile
146 print ' %s' % profile
144
147
145 def list_profile_dirs(self):
148 def list_profile_dirs(self):
146 profiles = self._list_bundled_profiles()
149 profiles = list_bundled_profiles()
147 if profiles:
150 if profiles:
148 print
151 print
149 print "Available profiles in IPython:"
152 print "Available profiles in IPython:"
@@ -153,13 +156,13 b' class ProfileList(Application):'
153 print " into your IPython directory (%s)," % self.ipython_dir
156 print " into your IPython directory (%s)," % self.ipython_dir
154 print " where you can customize it."
157 print " where you can customize it."
155
158
156 profiles = self._list_profiles_in(self.ipython_dir)
159 profiles = list_profiles_in(self.ipython_dir)
157 if profiles:
160 if profiles:
158 print
161 print
159 print "Available profiles in %s:" % self.ipython_dir
162 print "Available profiles in %s:" % self.ipython_dir
160 self._print_profiles(profiles)
163 self._print_profiles(profiles)
161
164
162 profiles = self._list_profiles_in(os.getcwdu())
165 profiles = list_profiles_in(os.getcwdu())
163 if profiles:
166 if profiles:
164 print
167 print
165 print "Available profiles in current directory (%s):" % os.getcwdu()
168 print "Available profiles in current directory (%s):" % os.getcwdu()
@@ -587,7 +587,6 b' class NotebookRootHandler(AuthenticatedHandler):'
587
587
588 @authenticate_unless_readonly
588 @authenticate_unless_readonly
589 def get(self):
589 def get(self):
590
591 nbm = self.application.notebook_manager
590 nbm = self.application.notebook_manager
592 files = nbm.list_notebooks()
591 files = nbm.list_notebooks()
593 self.finish(jsonapi.dumps(files))
592 self.finish(jsonapi.dumps(files))
@@ -661,6 +660,41 b' class NotebookCopyHandler(AuthenticatedHandler):'
661 mathjax_url=self.application.ipython_app.mathjax_url,
660 mathjax_url=self.application.ipython_app.mathjax_url,
662 )
661 )
663
662
663
664 #-----------------------------------------------------------------------------
665 # Cluster handlers
666 #-----------------------------------------------------------------------------
667
668
669 class MainClusterHandler(AuthenticatedHandler):
670
671 @web.authenticated
672 def get(self):
673 cm = self.application.cluster_manager
674 self.finish(jsonapi.dumps(cm.list_profiles()))
675
676
677 class ClusterProfileHandler(AuthenticatedHandler):
678
679 @web.authenticated
680 def get(self, profile):
681 cm = self.application.cluster_manager
682 self.finish(jsonapi.dumps(cm.profile_info(profile)))
683
684
685 class ClusterActionHandler(AuthenticatedHandler):
686
687 @web.authenticated
688 def post(self, profile, action):
689 cm = self.application.cluster_manager
690 if action == 'start':
691 n = int(self.get_argument('n', default=4))
692 cm.start_cluster(profile, n)
693 if action == 'stop':
694 cm.stop_cluster(profile)
695 self.finish()
696
697
664 #-----------------------------------------------------------------------------
698 #-----------------------------------------------------------------------------
665 # RST web service handlers
699 # RST web service handlers
666 #-----------------------------------------------------------------------------
700 #-----------------------------------------------------------------------------
@@ -49,9 +49,11 b' from .handlers import (LoginHandler, LogoutHandler,'
49 ProjectDashboardHandler, NewHandler, NamedNotebookHandler,
49 ProjectDashboardHandler, NewHandler, NamedNotebookHandler,
50 MainKernelHandler, KernelHandler, KernelActionHandler, IOPubHandler,
50 MainKernelHandler, KernelHandler, KernelActionHandler, IOPubHandler,
51 ShellHandler, NotebookRootHandler, NotebookHandler, NotebookCopyHandler,
51 ShellHandler, NotebookRootHandler, NotebookHandler, NotebookCopyHandler,
52 RSTHandler, AuthenticatedFileHandler, PrintNotebookHandler
52 RSTHandler, AuthenticatedFileHandler, PrintNotebookHandler,
53 MainClusterHandler, ClusterProfileHandler, ClusterActionHandler
53 )
54 )
54 from .notebookmanager import NotebookManager
55 from .notebookmanager import NotebookManager
56 from .clustermanager import ClusterManager
55
57
56 from IPython.config.application import catch_config_error, boolean_flag
58 from IPython.config.application import catch_config_error, boolean_flag
57 from IPython.core.application import BaseIPythonApplication
59 from IPython.core.application import BaseIPythonApplication
@@ -74,6 +76,9 b' from IPython.utils import py3compat'
74 _kernel_id_regex = r"(?P<kernel_id>\w+-\w+-\w+-\w+-\w+)"
76 _kernel_id_regex = r"(?P<kernel_id>\w+-\w+-\w+-\w+-\w+)"
75 _kernel_action_regex = r"(?P<action>restart|interrupt)"
77 _kernel_action_regex = r"(?P<action>restart|interrupt)"
76 _notebook_id_regex = r"(?P<notebook_id>\w+-\w+-\w+-\w+-\w+)"
78 _notebook_id_regex = r"(?P<notebook_id>\w+-\w+-\w+-\w+-\w+)"
79 _profile_regex = r"(?P<profile>[a-zA-Z0-9]+)"
80 _cluster_action_regex = r"(?P<action>start|stop)"
81
77
82
78 LOCALHOST = '127.0.0.1'
83 LOCALHOST = '127.0.0.1'
79
84
@@ -101,7 +106,8 b' def url_path_join(a,b):'
101
106
102 class NotebookWebApplication(web.Application):
107 class NotebookWebApplication(web.Application):
103
108
104 def __init__(self, ipython_app, kernel_manager, notebook_manager, log,
109 def __init__(self, ipython_app, kernel_manager, notebook_manager,
110 cluster_manager, log,
105 base_project_url, settings_overrides):
111 base_project_url, settings_overrides):
106 handlers = [
112 handlers = [
107 (r"/", ProjectDashboardHandler),
113 (r"/", ProjectDashboardHandler),
@@ -120,6 +126,9 b' class NotebookWebApplication(web.Application):'
120 (r"/notebooks/%s" % _notebook_id_regex, NotebookHandler),
126 (r"/notebooks/%s" % _notebook_id_regex, NotebookHandler),
121 (r"/rstservice/render", RSTHandler),
127 (r"/rstservice/render", RSTHandler),
122 (r"/files/(.*)", AuthenticatedFileHandler, {'path' : notebook_manager.notebook_dir}),
128 (r"/files/(.*)", AuthenticatedFileHandler, {'path' : notebook_manager.notebook_dir}),
129 (r"/clusters", MainClusterHandler),
130 (r"/clusters/%s/%s" % (_profile_regex, _cluster_action_regex), ClusterActionHandler),
131 (r"/clusters/%s" % _profile_regex, ClusterProfileHandler),
123 ]
132 ]
124 settings = dict(
133 settings = dict(
125 template_path=os.path.join(os.path.dirname(__file__), "templates"),
134 template_path=os.path.join(os.path.dirname(__file__), "templates"),
@@ -151,10 +160,11 b' class NotebookWebApplication(web.Application):'
151 super(NotebookWebApplication, self).__init__(new_handlers, **settings)
160 super(NotebookWebApplication, self).__init__(new_handlers, **settings)
152
161
153 self.kernel_manager = kernel_manager
162 self.kernel_manager = kernel_manager
154 self.log = log
155 self.notebook_manager = notebook_manager
163 self.notebook_manager = notebook_manager
164 self.cluster_manager = cluster_manager
156 self.ipython_app = ipython_app
165 self.ipython_app = ipython_app
157 self.read_only = self.ipython_app.read_only
166 self.read_only = self.ipython_app.read_only
167 self.log = log
158
168
159
169
160 #-----------------------------------------------------------------------------
170 #-----------------------------------------------------------------------------
@@ -395,6 +405,7 b' class NotebookApp(BaseIPythonApplication):'
395 )
405 )
396 self.notebook_manager = NotebookManager(config=self.config, log=self.log)
406 self.notebook_manager = NotebookManager(config=self.config, log=self.log)
397 self.notebook_manager.list_notebooks()
407 self.notebook_manager.list_notebooks()
408 self.cluster_manager = ClusterManager(config=self.config, log=self.log)
398
409
399 def init_logging(self):
410 def init_logging(self):
400 super(NotebookApp, self).init_logging()
411 super(NotebookApp, self).init_logging()
@@ -406,7 +417,8 b' class NotebookApp(BaseIPythonApplication):'
406 def init_webapp(self):
417 def init_webapp(self):
407 """initialize tornado webapp and httpserver"""
418 """initialize tornado webapp and httpserver"""
408 self.web_app = NotebookWebApplication(
419 self.web_app = NotebookWebApplication(
409 self, self.kernel_manager, self.notebook_manager, self.log,
420 self, self.kernel_manager, self.notebook_manager,
421 self.cluster_manager, self.log,
410 self.base_project_url, self.webapp_settings
422 self.base_project_url, self.webapp_settings
411 )
423 )
412 if self.certfile:
424 if self.certfile:
@@ -1164,14 +1164,16 b' class IPClusterLauncher(LocalProcessLauncher):'
1164 ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
1164 ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
1165 help="Popen command for ipcluster")
1165 help="Popen command for ipcluster")
1166 ipcluster_args = List(
1166 ipcluster_args = List(
1167 ['--clean-logs', '--log-to-file', '--log-level=%i'%logging.INFO], config=True,
1167 ['--clean-logs=True', '--log-to-file', '--log-level=%i'%logging.INFO], config=True,
1168 help="Command line arguments to pass to ipcluster.")
1168 help="Command line arguments to pass to ipcluster.")
1169 ipcluster_subcommand = Unicode('start')
1169 ipcluster_subcommand = Unicode('start')
1170 ipcluster_profile = Unicode('default')
1170 ipcluster_n = Integer(2)
1171 ipcluster_n = Integer(2)
1171
1172
1172 def find_args(self):
1173 def find_args(self):
1173 return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
1174 return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
1174 ['--n=%i'%self.ipcluster_n] + self.ipcluster_args
1175 ['--n=%i'%self.ipcluster_n, '--profile=%s'%self.ipcluster_profile] + \
1176 self.ipcluster_args
1175
1177
1176 def start(self):
1178 def start(self):
1177 return super(IPClusterLauncher, self).start()
1179 return super(IPClusterLauncher, self).start()
General Comments 0
You need to be logged in to leave comments. Login now