##// END OF EJS Templates
Finishing up help string work.
Brian E. Granger -
Show More
@@ -1,237 +1,238 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 An application for managing IPython profiles.
3 An application for managing IPython profiles.
4
4
5 To be invoked as the `ipython profile` subcommand.
5 To be invoked as the `ipython profile` subcommand.
6
6
7 Authors:
7 Authors:
8
8
9 * Min RK
9 * Min RK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 import logging
24 import logging
25 import os
25 import os
26
26
27 from IPython.config.application import Application, boolean_flag
27 from IPython.config.application import Application, boolean_flag
28 from IPython.core.application import (
28 from IPython.core.application import (
29 BaseIPythonApplication, base_flags, base_aliases
29 BaseIPythonApplication, base_flags, base_aliases
30 )
30 )
31 from IPython.core.profiledir import ProfileDir
31 from IPython.core.profiledir import ProfileDir
32 from IPython.utils.path import get_ipython_dir
32 from IPython.utils.path import get_ipython_dir
33 from IPython.utils.traitlets import Unicode, Bool, Dict
33 from IPython.utils.traitlets import Unicode, Bool, Dict
34
34
35 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
36 # Constants
36 # Constants
37 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
38
38
39 create_help = """Create an IPython profile by name
39 create_help = """Create an IPython profile by name
40
40
41 Create an ipython profile directory by its name or
41 Create an ipython profile directory by its name or
42 profile directory path. Profile directories contain
42 profile directory path. Profile directories contain
43 configuration, log and security related files and are named
43 configuration, log and security related files and are named
44 using the convention 'profile_<name>'. By default they are
44 using the convention 'profile_<name>'. By default they are
45 located in your ipython directory. Once created, you will
45 located in your ipython directory. Once created, you will
46 can edit the configuration files in the profile
46 can edit the configuration files in the profile
47 directory to configure IPython. Most users will create a
47 directory to configure IPython. Most users will create a
48 profile directory by name,
48 profile directory by name,
49 `ipython profile create myprofile`, which will put the directory
49 `ipython profile create myprofile`, which will put the directory
50 in `<ipython_dir>/profile_myprofile`.
50 in `<ipython_dir>/profile_myprofile`.
51 """
51 """
52 list_help = """List available IPython profiles
52 list_help = """List available IPython profiles
53
53
54 List all available profiles, by profile location, that can
54 List all available profiles, by profile location, that can
55 be found in the current working directly or in the ipython
55 be found in the current working directly or in the ipython
56 directory. Profile directories are named using the convention
56 directory. Profile directories are named using the convention
57 'profile_<profile>'.
57 'profile_<profile>'.
58 """
58 """
59 profile_help = """Manage IPython profiles
59 profile_help = """Manage IPython profiles
60
60
61 Profile directories contain
61 Profile directories contain
62 configuration, log and security related files and are named
62 configuration, log and security related files and are named
63 using the convention 'profile_<name>'. By default they are
63 using the convention 'profile_<name>'. By default they are
64 located in your ipython directory. You can create profiles
64 located in your ipython directory. You can create profiles
65 with `ipython profile create <name>`, or see the profiles you
65 with `ipython profile create <name>`, or see the profiles you
66 already have with `ipython profile list`
66 already have with `ipython profile list`
67
67
68 To get started configuring IPython, simply do:
68 To get started configuring IPython, simply do:
69
69
70 $> ipython profile create
70 $> ipython profile create
71
71
72 and IPython will create the default profile in <ipython_dir>/profile_default,
72 and IPython will create the default profile in <ipython_dir>/profile_default,
73 where you can edit ipython_config.py to start configuring IPython.
73 where you can edit ipython_config.py to start configuring IPython.
74
74
75 """
75 """
76
76
77 _list_examples = "ipython profile list # list all profiles"
77 _list_examples = "ipython profile list # list all profiles"
78
78
79 _create_examples = """
79 _create_examples = """
80 ipython profile create foo # create profile foo
80 ipython profile create foo # create profile foo w/ default config files
81 ipython profile create foo --init # create with default config files
81 ipython profile create foo --reset # restage default config files over current
82 ipython profile create foo --parallel # also stage parallel config files
82 """
83 """
83
84
84 _main_examples = """
85 _main_examples = """
85 ipython profile create -h # show the help string for the create subcommand
86 ipython profile create -h # show the help string for the create subcommand
86 ipython profile list -h # show the help string for the list subcommand
87 ipython profile list -h # show the help string for the list subcommand
87 """
88 """
88
89
89 #-----------------------------------------------------------------------------
90 #-----------------------------------------------------------------------------
90 # Profile Application Class (for `ipython profile` subcommand)
91 # Profile Application Class (for `ipython profile` subcommand)
91 #-----------------------------------------------------------------------------
92 #-----------------------------------------------------------------------------
92
93
93
94
94 class ProfileList(Application):
95 class ProfileList(Application):
95 name = u'ipython-profile'
96 name = u'ipython-profile'
96 description = list_help
97 description = list_help
97 examples = _list_examples
98 examples = _list_examples
98
99
99 aliases = Dict({
100 aliases = Dict({
100 'ipython-dir' : 'ProfileList.ipython_dir',
101 'ipython-dir' : 'ProfileList.ipython_dir',
101 'log-level' : 'Application.log_level',
102 'log-level' : 'Application.log_level',
102 })
103 })
103 flags = Dict(dict(
104 flags = Dict(dict(
104 debug = ({'Application' : {'log_level' : 0}},
105 debug = ({'Application' : {'log_level' : 0}},
105 "Set Application.log_level to 0, maximizing log output."
106 "Set Application.log_level to 0, maximizing log output."
106 )
107 )
107 ))
108 ))
108
109
109 ipython_dir = Unicode(get_ipython_dir(), config=True,
110 ipython_dir = Unicode(get_ipython_dir(), config=True,
110 help="""
111 help="""
111 The name of the IPython directory. This directory is used for logging
112 The name of the IPython directory. This directory is used for logging
112 configuration (through profiles), history storage, etc. The default
113 configuration (through profiles), history storage, etc. The default
113 is usually $HOME/.ipython. This options can also be specified through
114 is usually $HOME/.ipython. This options can also be specified through
114 the environment variable IPYTHON_DIR.
115 the environment variable IPYTHON_DIR.
115 """
116 """
116 )
117 )
117
118
118 def list_profile_dirs(self):
119 def list_profile_dirs(self):
119 # Find the search paths
120 # Find the search paths
120 paths = [os.getcwdu(), self.ipython_dir]
121 paths = [os.getcwdu(), self.ipython_dir]
121
122
122 self.log.warn('Searching for IPython profiles in paths: %r' % paths)
123 self.log.warn('Searching for IPython profiles in paths: %r' % paths)
123 for path in paths:
124 for path in paths:
124 files = os.listdir(path)
125 files = os.listdir(path)
125 for f in files:
126 for f in files:
126 full_path = os.path.join(path, f)
127 full_path = os.path.join(path, f)
127 if os.path.isdir(full_path) and f.startswith('profile_'):
128 if os.path.isdir(full_path) and f.startswith('profile_'):
128 profile = f.split('_',1)[-1]
129 profile = f.split('_',1)[-1]
129 start_cmd = 'ipython profile=%s' % profile
130 start_cmd = 'ipython profile=%s' % profile
130 print start_cmd + " ==> " + full_path
131 print start_cmd + " ==> " + full_path
131
132
132 def start(self):
133 def start(self):
133 self.list_profile_dirs()
134 self.list_profile_dirs()
134
135
135
136
136 create_flags = {}
137 create_flags = {}
137 create_flags.update(base_flags)
138 create_flags.update(base_flags)
138 create_flags.update(boolean_flag('reset', 'ProfileCreate.overwrite',
139 create_flags.update(boolean_flag('reset', 'ProfileCreate.overwrite',
139 "reset config files to defaults", "leave existing config files"))
140 "reset config files to defaults", "leave existing config files"))
140 create_flags.update(boolean_flag('parallel', 'ProfileCreate.parallel',
141 create_flags.update(boolean_flag('parallel', 'ProfileCreate.parallel',
141 "Include parallel computing config files",
142 "Include parallel computing config files",
142 "Don't include parallel computing config files"))
143 "Don't include parallel computing config files"))
143
144
144
145
145 class ProfileCreate(BaseIPythonApplication):
146 class ProfileCreate(BaseIPythonApplication):
146 name = u'ipython-profile'
147 name = u'ipython-profile'
147 description = create_help
148 description = create_help
148 examples = _create_examples
149 examples = _create_examples
149 auto_create = Bool(True, config=False)
150 auto_create = Bool(True, config=False)
150
151
151 def _copy_config_files_default(self):
152 def _copy_config_files_default(self):
152 return True
153 return True
153
154
154 parallel = Bool(False, config=True,
155 parallel = Bool(False, config=True,
155 help="whether to include parallel computing config files")
156 help="whether to include parallel computing config files")
156 def _parallel_changed(self, name, old, new):
157 def _parallel_changed(self, name, old, new):
157 parallel_files = [ 'ipcontroller_config.py',
158 parallel_files = [ 'ipcontroller_config.py',
158 'ipengine_config.py',
159 'ipengine_config.py',
159 'ipcluster_config.py'
160 'ipcluster_config.py'
160 ]
161 ]
161 if new:
162 if new:
162 for cf in parallel_files:
163 for cf in parallel_files:
163 self.config_files.append(cf)
164 self.config_files.append(cf)
164 else:
165 else:
165 for cf in parallel_files:
166 for cf in parallel_files:
166 if cf in self.config_files:
167 if cf in self.config_files:
167 self.config_files.remove(cf)
168 self.config_files.remove(cf)
168
169
169 def parse_command_line(self, argv):
170 def parse_command_line(self, argv):
170 super(ProfileCreate, self).parse_command_line(argv)
171 super(ProfileCreate, self).parse_command_line(argv)
171 # accept positional arg as profile name
172 # accept positional arg as profile name
172 if self.extra_args:
173 if self.extra_args:
173 self.profile = self.extra_args[0]
174 self.profile = self.extra_args[0]
174
175
175 flags = Dict(create_flags)
176 flags = Dict(create_flags)
176
177
177 classes = [ProfileDir]
178 classes = [ProfileDir]
178
179
179 def init_config_files(self):
180 def init_config_files(self):
180 super(ProfileCreate, self).init_config_files()
181 super(ProfileCreate, self).init_config_files()
181 # use local imports, since these classes may import from here
182 # use local imports, since these classes may import from here
182 from IPython.frontend.terminal.ipapp import TerminalIPythonApp
183 from IPython.frontend.terminal.ipapp import TerminalIPythonApp
183 apps = [TerminalIPythonApp]
184 apps = [TerminalIPythonApp]
184 try:
185 try:
185 from IPython.frontend.qt.console.qtconsoleapp import IPythonQtConsoleApp
186 from IPython.frontend.qt.console.qtconsoleapp import IPythonQtConsoleApp
186 except Exception:
187 except Exception:
187 # this should be ImportError, but under weird circumstances
188 # this should be ImportError, but under weird circumstances
188 # this might be an AttributeError, or possibly others
189 # this might be an AttributeError, or possibly others
189 # in any case, nothing should cause the profile creation to crash.
190 # in any case, nothing should cause the profile creation to crash.
190 pass
191 pass
191 else:
192 else:
192 apps.append(IPythonQtConsoleApp)
193 apps.append(IPythonQtConsoleApp)
193 if self.parallel:
194 if self.parallel:
194 from IPython.parallel.apps.ipcontrollerapp import IPControllerApp
195 from IPython.parallel.apps.ipcontrollerapp import IPControllerApp
195 from IPython.parallel.apps.ipengineapp import IPEngineApp
196 from IPython.parallel.apps.ipengineapp import IPEngineApp
196 from IPython.parallel.apps.ipclusterapp import IPClusterStart
197 from IPython.parallel.apps.ipclusterapp import IPClusterStart
197 from IPython.parallel.apps.iploggerapp import IPLoggerApp
198 from IPython.parallel.apps.iploggerapp import IPLoggerApp
198 apps.extend([
199 apps.extend([
199 IPControllerApp,
200 IPControllerApp,
200 IPEngineApp,
201 IPEngineApp,
201 IPClusterStart,
202 IPClusterStart,
202 IPLoggerApp,
203 IPLoggerApp,
203 ])
204 ])
204 for App in apps:
205 for App in apps:
205 app = App()
206 app = App()
206 app.config.update(self.config)
207 app.config.update(self.config)
207 app.log = self.log
208 app.log = self.log
208 app.overwrite = self.overwrite
209 app.overwrite = self.overwrite
209 app.copy_config_files=True
210 app.copy_config_files=True
210 app.profile = self.profile
211 app.profile = self.profile
211 app.init_profile_dir()
212 app.init_profile_dir()
212 app.init_config_files()
213 app.init_config_files()
213
214
214 def stage_default_config_file(self):
215 def stage_default_config_file(self):
215 pass
216 pass
216
217
217
218
218 class ProfileApp(Application):
219 class ProfileApp(Application):
219 name = u'ipython-profile'
220 name = u'ipython-profile'
220 description = profile_help
221 description = profile_help
221 examples = _main_examples
222 examples = _main_examples
222
223
223 subcommands = Dict(dict(
224 subcommands = Dict(dict(
224 create = (ProfileCreate, "Create a new profile dir with default config files"),
225 create = (ProfileCreate, "Create a new profile dir with default config files"),
225 list = (ProfileList, "List existing profiles")
226 list = (ProfileList, "List existing profiles")
226 ))
227 ))
227
228
228 def start(self):
229 def start(self):
229 if self.subapp is None:
230 if self.subapp is None:
230 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
231 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
231 print
232 print
232 self.print_description()
233 self.print_description()
233 self.print_subcommands()
234 self.print_subcommands()
234 self.exit(1)
235 self.exit(1)
235 else:
236 else:
236 return self.subapp.start()
237 return self.subapp.start()
237
238
@@ -1,375 +1,378 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The :class:`~IPython.core.application.Application` object for the command
4 The :class:`~IPython.core.application.Application` object for the command
5 line :command:`ipython` program.
5 line :command:`ipython` program.
6
6
7 Authors
7 Authors
8 -------
8 -------
9
9
10 * Brian Granger
10 * Brian Granger
11 * Fernando Perez
11 * Fernando Perez
12 * Min Ragan-Kelley
12 * Min Ragan-Kelley
13 """
13 """
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Copyright (C) 2008-2010 The IPython Development Team
16 # Copyright (C) 2008-2010 The IPython Development Team
17 #
17 #
18 # Distributed under the terms of the BSD License. The full license is in
18 # Distributed under the terms of the BSD License. The full license is in
19 # the file COPYING, distributed as part of this software.
19 # the file COPYING, distributed as part of this software.
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23 # Imports
23 # Imports
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25
25
26 from __future__ import absolute_import
26 from __future__ import absolute_import
27
27
28 import logging
28 import logging
29 import os
29 import os
30 import sys
30 import sys
31
31
32 from IPython.config.loader import (
32 from IPython.config.loader import (
33 Config, PyFileConfigLoader
33 Config, PyFileConfigLoader
34 )
34 )
35 from IPython.config.application import boolean_flag
35 from IPython.config.application import boolean_flag
36 from IPython.core import release
36 from IPython.core import release
37 from IPython.core import usage
37 from IPython.core import usage
38 from IPython.core.crashhandler import CrashHandler
38 from IPython.core.crashhandler import CrashHandler
39 from IPython.core.formatters import PlainTextFormatter
39 from IPython.core.formatters import PlainTextFormatter
40 from IPython.core.application import (
40 from IPython.core.application import (
41 ProfileDir, BaseIPythonApplication, base_flags, base_aliases
41 ProfileDir, BaseIPythonApplication, base_flags, base_aliases
42 )
42 )
43 from IPython.core.shellapp import (
43 from IPython.core.shellapp import (
44 InteractiveShellApp, shell_flags, shell_aliases
44 InteractiveShellApp, shell_flags, shell_aliases
45 )
45 )
46 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
46 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
47 from IPython.lib import inputhook
47 from IPython.lib import inputhook
48 from IPython.utils import warn
48 from IPython.utils import warn
49 from IPython.utils.path import get_ipython_dir, check_for_old_config
49 from IPython.utils.path import get_ipython_dir, check_for_old_config
50 from IPython.utils.traitlets import (
50 from IPython.utils.traitlets import (
51 Bool, Dict, CaselessStrEnum
51 Bool, Dict, CaselessStrEnum
52 )
52 )
53
53
54 #-----------------------------------------------------------------------------
54 #-----------------------------------------------------------------------------
55 # Globals, utilities and helpers
55 # Globals, utilities and helpers
56 #-----------------------------------------------------------------------------
56 #-----------------------------------------------------------------------------
57
57
58 #: The default config file name for this application.
58 #: The default config file name for this application.
59 default_config_file_name = u'ipython_config.py'
59 default_config_file_name = u'ipython_config.py'
60
60
61 _examples = """
61 _examples = """
62 ipython --pylab # start in pylab mode
62 ipython --pylab # start in pylab mode
63 ipython --pylab=qt # start in pylab mode with the qt4 backend
63 ipython --pylab=qt # start in pylab mode with the qt4 backend
64 ipython --log_level=DEBUG # set logging to DEBUG
64 ipython --log_level=DEBUG # set logging to DEBUG
65 ipython --profile=foo # start with profile foo
65 ipython --profile=foo # start with profile foo
66
66 ipython qtconsole # start the qtconsole GUI application
67 ipython qtconsole # start the qtconsole GUI application
67 ipython profile -h # show the help string for the profile subcmd
68 ipython qtconsole -h # show the help string for the qtconsole subcmd
68 ipython qtconsole -h # show the help string for the qtconsole subcmd
69
70 ipython profile create foo # create profile foo w/ default config files
71 ipython profile -h # show the help string for the profile subcmd
69 """
72 """
70
73
71 #-----------------------------------------------------------------------------
74 #-----------------------------------------------------------------------------
72 # Crash handler for this application
75 # Crash handler for this application
73 #-----------------------------------------------------------------------------
76 #-----------------------------------------------------------------------------
74
77
75 class IPAppCrashHandler(CrashHandler):
78 class IPAppCrashHandler(CrashHandler):
76 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
79 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
77
80
78 def __init__(self, app):
81 def __init__(self, app):
79 contact_name = release.authors['Fernando'][0]
82 contact_name = release.authors['Fernando'][0]
80 contact_email = release.authors['Fernando'][1]
83 contact_email = release.authors['Fernando'][1]
81 bug_tracker = 'http://github.com/ipython/ipython/issues'
84 bug_tracker = 'http://github.com/ipython/ipython/issues'
82 super(IPAppCrashHandler,self).__init__(
85 super(IPAppCrashHandler,self).__init__(
83 app, contact_name, contact_email, bug_tracker
86 app, contact_name, contact_email, bug_tracker
84 )
87 )
85
88
86 def make_report(self,traceback):
89 def make_report(self,traceback):
87 """Return a string containing a crash report."""
90 """Return a string containing a crash report."""
88
91
89 sec_sep = self.section_sep
92 sec_sep = self.section_sep
90 # Start with parent report
93 # Start with parent report
91 report = [super(IPAppCrashHandler, self).make_report(traceback)]
94 report = [super(IPAppCrashHandler, self).make_report(traceback)]
92 # Add interactive-specific info we may have
95 # Add interactive-specific info we may have
93 rpt_add = report.append
96 rpt_add = report.append
94 try:
97 try:
95 rpt_add(sec_sep+"History of session input:")
98 rpt_add(sec_sep+"History of session input:")
96 for line in self.app.shell.user_ns['_ih']:
99 for line in self.app.shell.user_ns['_ih']:
97 rpt_add(line)
100 rpt_add(line)
98 rpt_add('\n*** Last line of input (may not be in above history):\n')
101 rpt_add('\n*** Last line of input (may not be in above history):\n')
99 rpt_add(self.app.shell._last_input_line+'\n')
102 rpt_add(self.app.shell._last_input_line+'\n')
100 except:
103 except:
101 pass
104 pass
102
105
103 return ''.join(report)
106 return ''.join(report)
104
107
105 #-----------------------------------------------------------------------------
108 #-----------------------------------------------------------------------------
106 # Aliases and Flags
109 # Aliases and Flags
107 #-----------------------------------------------------------------------------
110 #-----------------------------------------------------------------------------
108 flags = dict(base_flags)
111 flags = dict(base_flags)
109 flags.update(shell_flags)
112 flags.update(shell_flags)
110 addflag = lambda *args: flags.update(boolean_flag(*args))
113 addflag = lambda *args: flags.update(boolean_flag(*args))
111 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
114 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
112 'Turn on auto editing of files with syntax errors.',
115 'Turn on auto editing of files with syntax errors.',
113 'Turn off auto editing of files with syntax errors.'
116 'Turn off auto editing of files with syntax errors.'
114 )
117 )
115 addflag('banner', 'TerminalIPythonApp.display_banner',
118 addflag('banner', 'TerminalIPythonApp.display_banner',
116 "Display a banner upon starting IPython.",
119 "Display a banner upon starting IPython.",
117 "Don't display a banner upon starting IPython."
120 "Don't display a banner upon starting IPython."
118 )
121 )
119 addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
122 addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
120 """Set to confirm when you try to exit IPython with an EOF (Control-D
123 """Set to confirm when you try to exit IPython with an EOF (Control-D
121 in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
124 in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
122 you can force a direct exit without any confirmation.""",
125 you can force a direct exit without any confirmation.""",
123 "Don't prompt the user when exiting."
126 "Don't prompt the user when exiting."
124 )
127 )
125 addflag('term-title', 'TerminalInteractiveShell.term_title',
128 addflag('term-title', 'TerminalInteractiveShell.term_title',
126 "Enable auto setting the terminal title.",
129 "Enable auto setting the terminal title.",
127 "Disable auto setting the terminal title."
130 "Disable auto setting the terminal title."
128 )
131 )
129 classic_config = Config()
132 classic_config = Config()
130 classic_config.InteractiveShell.cache_size = 0
133 classic_config.InteractiveShell.cache_size = 0
131 classic_config.PlainTextFormatter.pprint = False
134 classic_config.PlainTextFormatter.pprint = False
132 classic_config.InteractiveShell.prompt_in1 = '>>> '
135 classic_config.InteractiveShell.prompt_in1 = '>>> '
133 classic_config.InteractiveShell.prompt_in2 = '... '
136 classic_config.InteractiveShell.prompt_in2 = '... '
134 classic_config.InteractiveShell.prompt_out = ''
137 classic_config.InteractiveShell.prompt_out = ''
135 classic_config.InteractiveShell.separate_in = ''
138 classic_config.InteractiveShell.separate_in = ''
136 classic_config.InteractiveShell.separate_out = ''
139 classic_config.InteractiveShell.separate_out = ''
137 classic_config.InteractiveShell.separate_out2 = ''
140 classic_config.InteractiveShell.separate_out2 = ''
138 classic_config.InteractiveShell.colors = 'NoColor'
141 classic_config.InteractiveShell.colors = 'NoColor'
139 classic_config.InteractiveShell.xmode = 'Plain'
142 classic_config.InteractiveShell.xmode = 'Plain'
140
143
141 flags['classic']=(
144 flags['classic']=(
142 classic_config,
145 classic_config,
143 "Gives IPython a similar feel to the classic Python prompt."
146 "Gives IPython a similar feel to the classic Python prompt."
144 )
147 )
145 # # log doesn't make so much sense this way anymore
148 # # log doesn't make so much sense this way anymore
146 # paa('--log','-l',
149 # paa('--log','-l',
147 # action='store_true', dest='InteractiveShell.logstart',
150 # action='store_true', dest='InteractiveShell.logstart',
148 # help="Start logging to the default log file (./ipython_log.py).")
151 # help="Start logging to the default log file (./ipython_log.py).")
149 #
152 #
150 # # quick is harder to implement
153 # # quick is harder to implement
151 flags['quick']=(
154 flags['quick']=(
152 {'TerminalIPythonApp' : {'quick' : True}},
155 {'TerminalIPythonApp' : {'quick' : True}},
153 "Enable quick startup with no config files."
156 "Enable quick startup with no config files."
154 )
157 )
155
158
156 flags['i'] = (
159 flags['i'] = (
157 {'TerminalIPythonApp' : {'force_interact' : True}},
160 {'TerminalIPythonApp' : {'force_interact' : True}},
158 """also works as '-i'
161 """also works as '-i'
159 If running code from the command line, become interactive afterwards."""
162 If running code from the command line, become interactive afterwards."""
160 )
163 )
161 flags['pylab'] = (
164 flags['pylab'] = (
162 {'TerminalIPythonApp' : {'pylab' : 'auto'}},
165 {'TerminalIPythonApp' : {'pylab' : 'auto'}},
163 """Pre-load matplotlib and numpy for interactive use with
166 """Pre-load matplotlib and numpy for interactive use with
164 the default matplotlib backend."""
167 the default matplotlib backend."""
165 )
168 )
166
169
167 aliases = dict(base_aliases)
170 aliases = dict(base_aliases)
168 aliases.update(shell_aliases)
171 aliases.update(shell_aliases)
169
172
170 # it's possible we don't want short aliases for *all* of these:
173 # it's possible we don't want short aliases for *all* of these:
171 aliases.update(dict(
174 aliases.update(dict(
172 gui='TerminalIPythonApp.gui',
175 gui='TerminalIPythonApp.gui',
173 pylab='TerminalIPythonApp.pylab',
176 pylab='TerminalIPythonApp.pylab',
174 ))
177 ))
175
178
176 #-----------------------------------------------------------------------------
179 #-----------------------------------------------------------------------------
177 # Main classes and functions
180 # Main classes and functions
178 #-----------------------------------------------------------------------------
181 #-----------------------------------------------------------------------------
179
182
180
183
181 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
184 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
182 name = u'ipython'
185 name = u'ipython'
183 description = usage.cl_usage
186 description = usage.cl_usage
184 default_config_file_name = default_config_file_name
187 default_config_file_name = default_config_file_name
185 crash_handler_class = IPAppCrashHandler
188 crash_handler_class = IPAppCrashHandler
186 examples = _examples
189 examples = _examples
187
190
188 flags = Dict(flags)
191 flags = Dict(flags)
189 aliases = Dict(aliases)
192 aliases = Dict(aliases)
190 classes = [InteractiveShellApp, TerminalInteractiveShell, ProfileDir, PlainTextFormatter]
193 classes = [InteractiveShellApp, TerminalInteractiveShell, ProfileDir, PlainTextFormatter]
191 subcommands = Dict(dict(
194 subcommands = Dict(dict(
192 qtconsole=('IPython.frontend.qt.console.qtconsoleapp.IPythonQtConsoleApp',
195 qtconsole=('IPython.frontend.qt.console.qtconsoleapp.IPythonQtConsoleApp',
193 """Launch the IPython Qt Console."""
196 """Launch the IPython Qt Console."""
194 ),
197 ),
195 profile = ("IPython.core.profileapp.ProfileApp",
198 profile = ("IPython.core.profileapp.ProfileApp",
196 "Create and manage IPython profiles.")
199 "Create and manage IPython profiles.")
197 ))
200 ))
198
201
199 # *do* autocreate requested profile, but don't create the config file.
202 # *do* autocreate requested profile, but don't create the config file.
200 auto_create=Bool(True)
203 auto_create=Bool(True)
201 # configurables
204 # configurables
202 ignore_old_config=Bool(False, config=True,
205 ignore_old_config=Bool(False, config=True,
203 help="Suppress warning messages about legacy config files"
206 help="Suppress warning messages about legacy config files"
204 )
207 )
205 quick = Bool(False, config=True,
208 quick = Bool(False, config=True,
206 help="""Start IPython quickly by skipping the loading of config files."""
209 help="""Start IPython quickly by skipping the loading of config files."""
207 )
210 )
208 def _quick_changed(self, name, old, new):
211 def _quick_changed(self, name, old, new):
209 if new:
212 if new:
210 self.load_config_file = lambda *a, **kw: None
213 self.load_config_file = lambda *a, **kw: None
211 self.ignore_old_config=True
214 self.ignore_old_config=True
212
215
213 gui = CaselessStrEnum(('qt','wx','gtk'), config=True,
216 gui = CaselessStrEnum(('qt','wx','gtk'), config=True,
214 help="Enable GUI event loop integration ('qt', 'wx', 'gtk')."
217 help="Enable GUI event loop integration ('qt', 'wx', 'gtk')."
215 )
218 )
216 pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'auto'],
219 pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'auto'],
217 config=True,
220 config=True,
218 help="""Pre-load matplotlib and numpy for interactive use,
221 help="""Pre-load matplotlib and numpy for interactive use,
219 selecting a particular matplotlib backend and loop integration.
222 selecting a particular matplotlib backend and loop integration.
220 """
223 """
221 )
224 )
222 display_banner = Bool(True, config=True,
225 display_banner = Bool(True, config=True,
223 help="Whether to display a banner upon starting IPython."
226 help="Whether to display a banner upon starting IPython."
224 )
227 )
225
228
226 # if there is code of files to run from the cmd line, don't interact
229 # if there is code of files to run from the cmd line, don't interact
227 # unless the --i flag (App.force_interact) is true.
230 # unless the --i flag (App.force_interact) is true.
228 force_interact = Bool(False, config=True,
231 force_interact = Bool(False, config=True,
229 help="""If a command or file is given via the command-line,
232 help="""If a command or file is given via the command-line,
230 e.g. 'ipython foo.py"""
233 e.g. 'ipython foo.py"""
231 )
234 )
232 def _force_interact_changed(self, name, old, new):
235 def _force_interact_changed(self, name, old, new):
233 if new:
236 if new:
234 self.interact = True
237 self.interact = True
235
238
236 def _file_to_run_changed(self, name, old, new):
239 def _file_to_run_changed(self, name, old, new):
237 if new and not self.force_interact:
240 if new and not self.force_interact:
238 self.interact = False
241 self.interact = False
239 _code_to_run_changed = _file_to_run_changed
242 _code_to_run_changed = _file_to_run_changed
240
243
241 # internal, not-configurable
244 # internal, not-configurable
242 interact=Bool(True)
245 interact=Bool(True)
243
246
244
247
245 def parse_command_line(self, argv=None):
248 def parse_command_line(self, argv=None):
246 """override to allow old '-pylab' flag with deprecation warning"""
249 """override to allow old '-pylab' flag with deprecation warning"""
247 argv = sys.argv[1:] if argv is None else argv
250 argv = sys.argv[1:] if argv is None else argv
248
251
249 try:
252 try:
250 idx = argv.index('-pylab')
253 idx = argv.index('-pylab')
251 except ValueError:
254 except ValueError:
252 # `-pylab` not given, proceed as normal
255 # `-pylab` not given, proceed as normal
253 pass
256 pass
254 else:
257 else:
255 # deprecated `-pylab` given,
258 # deprecated `-pylab` given,
256 # warn and transform into current syntax
259 # warn and transform into current syntax
257 argv = list(argv) # copy, don't clobber
260 argv = list(argv) # copy, don't clobber
258 warn.warn("`-pylab` flag has been deprecated.\n"
261 warn.warn("`-pylab` flag has been deprecated.\n"
259 " Use `--pylab` instead, or `--pylab=foo` to specify a backend.")
262 " Use `--pylab` instead, or `--pylab=foo` to specify a backend.")
260 sub = '--pylab'
263 sub = '--pylab'
261 if len(argv) > idx+1:
264 if len(argv) > idx+1:
262 # check for gui arg, as in '-pylab qt'
265 # check for gui arg, as in '-pylab qt'
263 gui = argv[idx+1]
266 gui = argv[idx+1]
264 if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):
267 if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):
265 sub = '--pylab='+gui
268 sub = '--pylab='+gui
266 argv.pop(idx+1)
269 argv.pop(idx+1)
267 argv[idx] = sub
270 argv[idx] = sub
268
271
269 return super(TerminalIPythonApp, self).parse_command_line(argv)
272 return super(TerminalIPythonApp, self).parse_command_line(argv)
270
273
271 def initialize(self, argv=None):
274 def initialize(self, argv=None):
272 """Do actions after construct, but before starting the app."""
275 """Do actions after construct, but before starting the app."""
273 super(TerminalIPythonApp, self).initialize(argv)
276 super(TerminalIPythonApp, self).initialize(argv)
274 if self.subapp is not None:
277 if self.subapp is not None:
275 # don't bother initializing further, starting subapp
278 # don't bother initializing further, starting subapp
276 return
279 return
277 if not self.ignore_old_config:
280 if not self.ignore_old_config:
278 check_for_old_config(self.ipython_dir)
281 check_for_old_config(self.ipython_dir)
279 # print self.extra_args
282 # print self.extra_args
280 if self.extra_args:
283 if self.extra_args:
281 self.file_to_run = self.extra_args[0]
284 self.file_to_run = self.extra_args[0]
282 # create the shell
285 # create the shell
283 self.init_shell()
286 self.init_shell()
284 # and draw the banner
287 # and draw the banner
285 self.init_banner()
288 self.init_banner()
286 # Now a variety of things that happen after the banner is printed.
289 # Now a variety of things that happen after the banner is printed.
287 self.init_gui_pylab()
290 self.init_gui_pylab()
288 self.init_extensions()
291 self.init_extensions()
289 self.init_code()
292 self.init_code()
290
293
291 def init_shell(self):
294 def init_shell(self):
292 """initialize the InteractiveShell instance"""
295 """initialize the InteractiveShell instance"""
293 # I am a little hesitant to put these into InteractiveShell itself.
296 # I am a little hesitant to put these into InteractiveShell itself.
294 # But that might be the place for them
297 # But that might be the place for them
295 sys.path.insert(0, '')
298 sys.path.insert(0, '')
296
299
297 # Create an InteractiveShell instance.
300 # Create an InteractiveShell instance.
298 # shell.display_banner should always be False for the terminal
301 # shell.display_banner should always be False for the terminal
299 # based app, because we call shell.show_banner() by hand below
302 # based app, because we call shell.show_banner() by hand below
300 # so the banner shows *before* all extension loading stuff.
303 # so the banner shows *before* all extension loading stuff.
301 self.shell = TerminalInteractiveShell.instance(config=self.config,
304 self.shell = TerminalInteractiveShell.instance(config=self.config,
302 display_banner=False, profile_dir=self.profile_dir,
305 display_banner=False, profile_dir=self.profile_dir,
303 ipython_dir=self.ipython_dir)
306 ipython_dir=self.ipython_dir)
304
307
305 def init_banner(self):
308 def init_banner(self):
306 """optionally display the banner"""
309 """optionally display the banner"""
307 if self.display_banner and self.interact:
310 if self.display_banner and self.interact:
308 self.shell.show_banner()
311 self.shell.show_banner()
309 # Make sure there is a space below the banner.
312 # Make sure there is a space below the banner.
310 if self.log_level <= logging.INFO: print
313 if self.log_level <= logging.INFO: print
311
314
312
315
313 def init_gui_pylab(self):
316 def init_gui_pylab(self):
314 """Enable GUI event loop integration, taking pylab into account."""
317 """Enable GUI event loop integration, taking pylab into account."""
315 gui = self.gui
318 gui = self.gui
316
319
317 # Using `pylab` will also require gui activation, though which toolkit
320 # Using `pylab` will also require gui activation, though which toolkit
318 # to use may be chosen automatically based on mpl configuration.
321 # to use may be chosen automatically based on mpl configuration.
319 if self.pylab:
322 if self.pylab:
320 activate = self.shell.enable_pylab
323 activate = self.shell.enable_pylab
321 if self.pylab == 'auto':
324 if self.pylab == 'auto':
322 gui = None
325 gui = None
323 else:
326 else:
324 gui = self.pylab
327 gui = self.pylab
325 else:
328 else:
326 # Enable only GUI integration, no pylab
329 # Enable only GUI integration, no pylab
327 activate = inputhook.enable_gui
330 activate = inputhook.enable_gui
328
331
329 if gui or self.pylab:
332 if gui or self.pylab:
330 try:
333 try:
331 self.log.info("Enabling GUI event loop integration, "
334 self.log.info("Enabling GUI event loop integration, "
332 "toolkit=%s, pylab=%s" % (gui, self.pylab) )
335 "toolkit=%s, pylab=%s" % (gui, self.pylab) )
333 activate(gui)
336 activate(gui)
334 except:
337 except:
335 self.log.warn("Error in enabling GUI event loop integration:")
338 self.log.warn("Error in enabling GUI event loop integration:")
336 self.shell.showtraceback()
339 self.shell.showtraceback()
337
340
338 def start(self):
341 def start(self):
339 if self.subapp is not None:
342 if self.subapp is not None:
340 return self.subapp.start()
343 return self.subapp.start()
341 # perform any prexec steps:
344 # perform any prexec steps:
342 if self.interact:
345 if self.interact:
343 self.log.debug("Starting IPython's mainloop...")
346 self.log.debug("Starting IPython's mainloop...")
344 self.shell.mainloop()
347 self.shell.mainloop()
345 else:
348 else:
346 self.log.debug("IPython not interactive...")
349 self.log.debug("IPython not interactive...")
347
350
348
351
349 def load_default_config(ipython_dir=None):
352 def load_default_config(ipython_dir=None):
350 """Load the default config file from the default ipython_dir.
353 """Load the default config file from the default ipython_dir.
351
354
352 This is useful for embedded shells.
355 This is useful for embedded shells.
353 """
356 """
354 if ipython_dir is None:
357 if ipython_dir is None:
355 ipython_dir = get_ipython_dir()
358 ipython_dir = get_ipython_dir()
356 profile_dir = os.path.join(ipython_dir, 'profile_default')
359 profile_dir = os.path.join(ipython_dir, 'profile_default')
357 cl = PyFileConfigLoader(default_config_file_name, profile_dir)
360 cl = PyFileConfigLoader(default_config_file_name, profile_dir)
358 try:
361 try:
359 config = cl.load_config()
362 config = cl.load_config()
360 except IOError:
363 except IOError:
361 # no config found
364 # no config found
362 config = Config()
365 config = Config()
363 return config
366 return config
364
367
365
368
366 def launch_new_instance():
369 def launch_new_instance():
367 """Create and run a full blown IPython instance"""
370 """Create and run a full blown IPython instance"""
368 app = TerminalIPythonApp.instance()
371 app = TerminalIPythonApp.instance()
369 app.initialize()
372 app.initialize()
370 app.start()
373 app.start()
371
374
372
375
373 if __name__ == '__main__':
376 if __name__ == '__main__':
374 launch_new_instance()
377 launch_new_instance()
375
378
@@ -1,482 +1,484 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The ipcluster application.
4 The ipcluster application.
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 import errno
24 import errno
25 import logging
25 import logging
26 import os
26 import os
27 import re
27 import re
28 import signal
28 import signal
29
29
30 from subprocess import check_call, CalledProcessError, PIPE
30 from subprocess import check_call, CalledProcessError, PIPE
31 import zmq
31 import zmq
32 from zmq.eventloop import ioloop
32 from zmq.eventloop import ioloop
33
33
34 from IPython.config.application import Application, boolean_flag
34 from IPython.config.application import Application, boolean_flag
35 from IPython.config.loader import Config
35 from IPython.config.loader import Config
36 from IPython.core.application import BaseIPythonApplication
36 from IPython.core.application import BaseIPythonApplication
37 from IPython.core.profiledir import ProfileDir
37 from IPython.core.profiledir import ProfileDir
38 from IPython.utils.daemonize import daemonize
38 from IPython.utils.daemonize import daemonize
39 from IPython.utils.importstring import import_item
39 from IPython.utils.importstring import import_item
40 from IPython.utils.traitlets import (Int, Unicode, Bool, CFloat, Dict, List,
40 from IPython.utils.traitlets import (Int, Unicode, Bool, CFloat, Dict, List,
41 DottedObjectName)
41 DottedObjectName)
42
42
43 from IPython.parallel.apps.baseapp import (
43 from IPython.parallel.apps.baseapp import (
44 BaseParallelApplication,
44 BaseParallelApplication,
45 PIDFileError,
45 PIDFileError,
46 base_flags, base_aliases
46 base_flags, base_aliases
47 )
47 )
48
48
49
49
50 #-----------------------------------------------------------------------------
50 #-----------------------------------------------------------------------------
51 # Module level variables
51 # Module level variables
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53
53
54
54
55 default_config_file_name = u'ipcluster_config.py'
55 default_config_file_name = u'ipcluster_config.py'
56
56
57
57
58 _description = """Start an IPython cluster for parallel computing.
58 _description = """Start an IPython cluster for parallel computing.
59
59
60 An IPython cluster consists of 1 controller and 1 or more engines.
60 An IPython cluster consists of 1 controller and 1 or more engines.
61 This command automates the startup of these processes using a wide
61 This command automates the startup of these processes using a wide
62 range of startup methods (SSH, local processes, PBS, mpiexec,
62 range of startup methods (SSH, local processes, PBS, mpiexec,
63 Windows HPC Server 2008). To start a cluster with 4 engines on your
63 Windows HPC Server 2008). To start a cluster with 4 engines on your
64 local host simply do 'ipcluster start --n=4'. For more complex usage
64 local host simply do 'ipcluster start --n=4'. For more complex usage
65 you will typically do 'ipython create mycluster --parallel', then edit
65 you will typically do 'ipython create mycluster --parallel', then edit
66 configuration files, followed by 'ipcluster start --profile=mycluster --n=4'.
66 configuration files, followed by 'ipcluster start --profile=mycluster --n=4'.
67 """
67 """
68
68
69 _main_examples = """
69 _main_examples = """
70 ipcluster start --n=4 # start a 4 node cluster on localhost
70 ipcluster start -h # show the help string for the start subcmd
71 ipcluster start -h # show the help string for the start subcmd
72
71 ipcluster stop -h # show the help string for the stop subcmd
73 ipcluster stop -h # show the help string for the stop subcmd
72 ipcluster engines -h # show the help string for the engines subcmd
74 ipcluster engines -h # show the help string for the engines subcmd
73 """
75 """
74
76
75 _start_examples = """
77 _start_examples = """
76 ipython profile create mycluster --parallel # create mycluster profile
78 ipython profile create mycluster --parallel # create mycluster profile
77 ipcluster start --profile=mycluster --n=4 # start mycluster with 4 nodes
79 ipcluster start --profile=mycluster --n=4 # start mycluster with 4 nodes
78 """
80 """
79
81
80 _stop_examples = """
82 _stop_examples = """
81 ipcluster stop --profile=mycluster # stop a running cluster by profile name
83 ipcluster stop --profile=mycluster # stop a running cluster by profile name
82 """
84 """
83
85
84 _engines_examples = """
86 _engines_examples = """
85 ipcluster engines --profile=mycluster --n=4 # start 4 engines only
87 ipcluster engines --profile=mycluster --n=4 # start 4 engines only
86 """
88 """
87
89
88
90
89 # Exit codes for ipcluster
91 # Exit codes for ipcluster
90
92
91 # This will be the exit code if the ipcluster appears to be running because
93 # This will be the exit code if the ipcluster appears to be running because
92 # a .pid file exists
94 # a .pid file exists
93 ALREADY_STARTED = 10
95 ALREADY_STARTED = 10
94
96
95
97
96 # This will be the exit code if ipcluster stop is run, but there is not .pid
98 # This will be the exit code if ipcluster stop is run, but there is not .pid
97 # file to be found.
99 # file to be found.
98 ALREADY_STOPPED = 11
100 ALREADY_STOPPED = 11
99
101
100 # This will be the exit code if ipcluster engines is run, but there is not .pid
102 # This will be the exit code if ipcluster engines is run, but there is not .pid
101 # file to be found.
103 # file to be found.
102 NO_CLUSTER = 12
104 NO_CLUSTER = 12
103
105
104
106
105 #-----------------------------------------------------------------------------
107 #-----------------------------------------------------------------------------
106 # Main application
108 # Main application
107 #-----------------------------------------------------------------------------
109 #-----------------------------------------------------------------------------
108 start_help = """Start an IPython cluster for parallel computing
110 start_help = """Start an IPython cluster for parallel computing
109
111
110 Start an ipython cluster by its profile name or cluster
112 Start an ipython cluster by its profile name or cluster
111 directory. Cluster directories contain configuration, log and
113 directory. Cluster directories contain configuration, log and
112 security related files and are named using the convention
114 security related files and are named using the convention
113 'profile_<name>' and should be creating using the 'start'
115 'profile_<name>' and should be creating using the 'start'
114 subcommand of 'ipcluster'. If your cluster directory is in
116 subcommand of 'ipcluster'. If your cluster directory is in
115 the cwd or the ipython directory, you can simply refer to it
117 the cwd or the ipython directory, you can simply refer to it
116 using its profile name, 'ipcluster start n=4 profile=<profile>`,
118 using its profile name, 'ipcluster start --n=4 --profile=<profile>`,
117 otherwise use the 'profile_dir' option.
119 otherwise use the 'profile-dir' option.
118 """
120 """
119 stop_help = """Stop a running IPython cluster
121 stop_help = """Stop a running IPython cluster
120
122
121 Stop a running ipython cluster by its profile name or cluster
123 Stop a running ipython cluster by its profile name or cluster
122 directory. Cluster directories are named using the convention
124 directory. Cluster directories are named using the convention
123 'profile_<name>'. If your cluster directory is in
125 'profile_<name>'. If your cluster directory is in
124 the cwd or the ipython directory, you can simply refer to it
126 the cwd or the ipython directory, you can simply refer to it
125 using its profile name, 'ipcluster stop profile=<profile>`, otherwise
127 using its profile name, 'ipcluster stop --profile=<profile>`, otherwise
126 use the 'profile_dir' option.
128 use the '--profile-dir' option.
127 """
129 """
128 engines_help = """Start engines connected to an existing IPython cluster
130 engines_help = """Start engines connected to an existing IPython cluster
129
131
130 Start one or more engines to connect to an existing Cluster
132 Start one or more engines to connect to an existing Cluster
131 by profile name or cluster directory.
133 by profile name or cluster directory.
132 Cluster directories contain configuration, log and
134 Cluster directories contain configuration, log and
133 security related files and are named using the convention
135 security related files and are named using the convention
134 'profile_<name>' and should be creating using the 'start'
136 'profile_<name>' and should be creating using the 'start'
135 subcommand of 'ipcluster'. If your cluster directory is in
137 subcommand of 'ipcluster'. If your cluster directory is in
136 the cwd or the ipython directory, you can simply refer to it
138 the cwd or the ipython directory, you can simply refer to it
137 using its profile name, 'ipcluster engines n=4 profile=<profile>`,
139 using its profile name, 'ipcluster engines --n=4 --profile=<profile>`,
138 otherwise use the 'profile_dir' option.
140 otherwise use the 'profile-dir' option.
139 """
141 """
140 stop_aliases = dict(
142 stop_aliases = dict(
141 signal='IPClusterStop.signal',
143 signal='IPClusterStop.signal',
142 )
144 )
143 stop_aliases.update(base_aliases)
145 stop_aliases.update(base_aliases)
144
146
145 class IPClusterStop(BaseParallelApplication):
147 class IPClusterStop(BaseParallelApplication):
146 name = u'ipcluster'
148 name = u'ipcluster'
147 description = stop_help
149 description = stop_help
148 examples = _stop_examples
150 examples = _stop_examples
149 config_file_name = Unicode(default_config_file_name)
151 config_file_name = Unicode(default_config_file_name)
150
152
151 signal = Int(signal.SIGINT, config=True,
153 signal = Int(signal.SIGINT, config=True,
152 help="signal to use for stopping processes.")
154 help="signal to use for stopping processes.")
153
155
154 aliases = Dict(stop_aliases)
156 aliases = Dict(stop_aliases)
155
157
156 def start(self):
158 def start(self):
157 """Start the app for the stop subcommand."""
159 """Start the app for the stop subcommand."""
158 try:
160 try:
159 pid = self.get_pid_from_file()
161 pid = self.get_pid_from_file()
160 except PIDFileError:
162 except PIDFileError:
161 self.log.critical(
163 self.log.critical(
162 'Could not read pid file, cluster is probably not running.'
164 'Could not read pid file, cluster is probably not running.'
163 )
165 )
164 # Here I exit with a unusual exit status that other processes
166 # Here I exit with a unusual exit status that other processes
165 # can watch for to learn how I existed.
167 # can watch for to learn how I existed.
166 self.remove_pid_file()
168 self.remove_pid_file()
167 self.exit(ALREADY_STOPPED)
169 self.exit(ALREADY_STOPPED)
168
170
169 if not self.check_pid(pid):
171 if not self.check_pid(pid):
170 self.log.critical(
172 self.log.critical(
171 'Cluster [pid=%r] is not running.' % pid
173 'Cluster [pid=%r] is not running.' % pid
172 )
174 )
173 self.remove_pid_file()
175 self.remove_pid_file()
174 # Here I exit with a unusual exit status that other processes
176 # Here I exit with a unusual exit status that other processes
175 # can watch for to learn how I existed.
177 # can watch for to learn how I existed.
176 self.exit(ALREADY_STOPPED)
178 self.exit(ALREADY_STOPPED)
177
179
178 elif os.name=='posix':
180 elif os.name=='posix':
179 sig = self.signal
181 sig = self.signal
180 self.log.info(
182 self.log.info(
181 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
183 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
182 )
184 )
183 try:
185 try:
184 os.kill(pid, sig)
186 os.kill(pid, sig)
185 except OSError:
187 except OSError:
186 self.log.error("Stopping cluster failed, assuming already dead.",
188 self.log.error("Stopping cluster failed, assuming already dead.",
187 exc_info=True)
189 exc_info=True)
188 self.remove_pid_file()
190 self.remove_pid_file()
189 elif os.name=='nt':
191 elif os.name=='nt':
190 try:
192 try:
191 # kill the whole tree
193 # kill the whole tree
192 p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
194 p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
193 except (CalledProcessError, OSError):
195 except (CalledProcessError, OSError):
194 self.log.error("Stopping cluster failed, assuming already dead.",
196 self.log.error("Stopping cluster failed, assuming already dead.",
195 exc_info=True)
197 exc_info=True)
196 self.remove_pid_file()
198 self.remove_pid_file()
197
199
198 engine_aliases = {}
200 engine_aliases = {}
199 engine_aliases.update(base_aliases)
201 engine_aliases.update(base_aliases)
200 engine_aliases.update(dict(
202 engine_aliases.update(dict(
201 n='IPClusterEngines.n',
203 n='IPClusterEngines.n',
202 engines = 'IPClusterEngines.engine_launcher_class',
204 engines = 'IPClusterEngines.engine_launcher_class',
203 daemonize = 'IPClusterEngines.daemonize',
205 daemonize = 'IPClusterEngines.daemonize',
204 ))
206 ))
205 engine_flags = {}
207 engine_flags = {}
206 engine_flags.update(base_flags)
208 engine_flags.update(base_flags)
207
209
208 engine_flags.update(dict(
210 engine_flags.update(dict(
209 daemonize=(
211 daemonize=(
210 {'IPClusterEngines' : {'daemonize' : True}},
212 {'IPClusterEngines' : {'daemonize' : True}},
211 """run the cluster into the background (not available on Windows)""",
213 """run the cluster into the background (not available on Windows)""",
212 )
214 )
213 ))
215 ))
214 class IPClusterEngines(BaseParallelApplication):
216 class IPClusterEngines(BaseParallelApplication):
215
217
216 name = u'ipcluster'
218 name = u'ipcluster'
217 description = engines_help
219 description = engines_help
218 examples = _engines_examples
220 examples = _engines_examples
219 usage = None
221 usage = None
220 config_file_name = Unicode(default_config_file_name)
222 config_file_name = Unicode(default_config_file_name)
221 default_log_level = logging.INFO
223 default_log_level = logging.INFO
222 classes = List()
224 classes = List()
223 def _classes_default(self):
225 def _classes_default(self):
224 from IPython.parallel.apps import launcher
226 from IPython.parallel.apps import launcher
225 launchers = launcher.all_launchers
227 launchers = launcher.all_launchers
226 eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
228 eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
227 return [ProfileDir]+eslaunchers
229 return [ProfileDir]+eslaunchers
228
230
229 n = Int(2, config=True,
231 n = Int(2, config=True,
230 help="The number of engines to start.")
232 help="The number of engines to start.")
231
233
232 engine_launcher_class = DottedObjectName('LocalEngineSetLauncher',
234 engine_launcher_class = DottedObjectName('LocalEngineSetLauncher',
233 config=True,
235 config=True,
234 help="The class for launching a set of Engines."
236 help="The class for launching a set of Engines."
235 )
237 )
236 daemonize = Bool(False, config=True,
238 daemonize = Bool(False, config=True,
237 help="""Daemonize the ipcluster program. This implies --log-to-file.
239 help="""Daemonize the ipcluster program. This implies --log-to-file.
238 Not available on Windows.
240 Not available on Windows.
239 """)
241 """)
240
242
241 def _daemonize_changed(self, name, old, new):
243 def _daemonize_changed(self, name, old, new):
242 if new:
244 if new:
243 self.log_to_file = True
245 self.log_to_file = True
244
246
245 aliases = Dict(engine_aliases)
247 aliases = Dict(engine_aliases)
246 flags = Dict(engine_flags)
248 flags = Dict(engine_flags)
247 _stopping = False
249 _stopping = False
248
250
249 def initialize(self, argv=None):
251 def initialize(self, argv=None):
250 super(IPClusterEngines, self).initialize(argv)
252 super(IPClusterEngines, self).initialize(argv)
251 self.init_signal()
253 self.init_signal()
252 self.init_launchers()
254 self.init_launchers()
253
255
254 def init_launchers(self):
256 def init_launchers(self):
255 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
257 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
256 self.engine_launcher.on_stop(lambda r: self.loop.stop())
258 self.engine_launcher.on_stop(lambda r: self.loop.stop())
257
259
258 def init_signal(self):
260 def init_signal(self):
259 # Setup signals
261 # Setup signals
260 signal.signal(signal.SIGINT, self.sigint_handler)
262 signal.signal(signal.SIGINT, self.sigint_handler)
261
263
262 def build_launcher(self, clsname):
264 def build_launcher(self, clsname):
263 """import and instantiate a Launcher based on importstring"""
265 """import and instantiate a Launcher based on importstring"""
264 if '.' not in clsname:
266 if '.' not in clsname:
265 # not a module, presume it's the raw name in apps.launcher
267 # not a module, presume it's the raw name in apps.launcher
266 clsname = 'IPython.parallel.apps.launcher.'+clsname
268 clsname = 'IPython.parallel.apps.launcher.'+clsname
267 # print repr(clsname)
269 # print repr(clsname)
268 klass = import_item(clsname)
270 klass = import_item(clsname)
269
271
270 launcher = klass(
272 launcher = klass(
271 work_dir=self.profile_dir.location, config=self.config, log=self.log
273 work_dir=self.profile_dir.location, config=self.config, log=self.log
272 )
274 )
273 return launcher
275 return launcher
274
276
275 def start_engines(self):
277 def start_engines(self):
276 self.log.info("Starting %i engines"%self.n)
278 self.log.info("Starting %i engines"%self.n)
277 self.engine_launcher.start(
279 self.engine_launcher.start(
278 self.n,
280 self.n,
279 self.profile_dir.location
281 self.profile_dir.location
280 )
282 )
281
283
282 def stop_engines(self):
284 def stop_engines(self):
283 self.log.info("Stopping Engines...")
285 self.log.info("Stopping Engines...")
284 if self.engine_launcher.running:
286 if self.engine_launcher.running:
285 d = self.engine_launcher.stop()
287 d = self.engine_launcher.stop()
286 return d
288 return d
287 else:
289 else:
288 return None
290 return None
289
291
290 def stop_launchers(self, r=None):
292 def stop_launchers(self, r=None):
291 if not self._stopping:
293 if not self._stopping:
292 self._stopping = True
294 self._stopping = True
293 self.log.error("IPython cluster: stopping")
295 self.log.error("IPython cluster: stopping")
294 self.stop_engines()
296 self.stop_engines()
295 # Wait a few seconds to let things shut down.
297 # Wait a few seconds to let things shut down.
296 dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
298 dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
297 dc.start()
299 dc.start()
298
300
299 def sigint_handler(self, signum, frame):
301 def sigint_handler(self, signum, frame):
300 self.log.debug("SIGINT received, stopping launchers...")
302 self.log.debug("SIGINT received, stopping launchers...")
301 self.stop_launchers()
303 self.stop_launchers()
302
304
303 def start_logging(self):
305 def start_logging(self):
304 # Remove old log files of the controller and engine
306 # Remove old log files of the controller and engine
305 if self.clean_logs:
307 if self.clean_logs:
306 log_dir = self.profile_dir.log_dir
308 log_dir = self.profile_dir.log_dir
307 for f in os.listdir(log_dir):
309 for f in os.listdir(log_dir):
308 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
310 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
309 os.remove(os.path.join(log_dir, f))
311 os.remove(os.path.join(log_dir, f))
310 # This will remove old log files for ipcluster itself
312 # This will remove old log files for ipcluster itself
311 # super(IPBaseParallelApplication, self).start_logging()
313 # super(IPBaseParallelApplication, self).start_logging()
312
314
313 def start(self):
315 def start(self):
314 """Start the app for the engines subcommand."""
316 """Start the app for the engines subcommand."""
315 self.log.info("IPython cluster: started")
317 self.log.info("IPython cluster: started")
316 # First see if the cluster is already running
318 # First see if the cluster is already running
317
319
318 # Now log and daemonize
320 # Now log and daemonize
319 self.log.info(
321 self.log.info(
320 'Starting engines with [daemon=%r]' % self.daemonize
322 'Starting engines with [daemon=%r]' % self.daemonize
321 )
323 )
322 # TODO: Get daemonize working on Windows or as a Windows Server.
324 # TODO: Get daemonize working on Windows or as a Windows Server.
323 if self.daemonize:
325 if self.daemonize:
324 if os.name=='posix':
326 if os.name=='posix':
325 daemonize()
327 daemonize()
326
328
327 dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
329 dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
328 dc.start()
330 dc.start()
329 # Now write the new pid file AFTER our new forked pid is active.
331 # Now write the new pid file AFTER our new forked pid is active.
330 # self.write_pid_file()
332 # self.write_pid_file()
331 try:
333 try:
332 self.loop.start()
334 self.loop.start()
333 except KeyboardInterrupt:
335 except KeyboardInterrupt:
334 pass
336 pass
335 except zmq.ZMQError as e:
337 except zmq.ZMQError as e:
336 if e.errno == errno.EINTR:
338 if e.errno == errno.EINTR:
337 pass
339 pass
338 else:
340 else:
339 raise
341 raise
340
342
341 start_aliases = {}
343 start_aliases = {}
342 start_aliases.update(engine_aliases)
344 start_aliases.update(engine_aliases)
343 start_aliases.update(dict(
345 start_aliases.update(dict(
344 delay='IPClusterStart.delay',
346 delay='IPClusterStart.delay',
345 controller = 'IPClusterStart.controller_launcher_class',
347 controller = 'IPClusterStart.controller_launcher_class',
346 ))
348 ))
347 start_aliases['clean-logs'] = 'IPClusterStart.clean_logs'
349 start_aliases['clean-logs'] = 'IPClusterStart.clean_logs'
348
350
349 class IPClusterStart(IPClusterEngines):
351 class IPClusterStart(IPClusterEngines):
350
352
351 name = u'ipcluster'
353 name = u'ipcluster'
352 description = start_help
354 description = start_help
353 examples = _start_examples
355 examples = _start_examples
354 default_log_level = logging.INFO
356 default_log_level = logging.INFO
355 auto_create = Bool(True, config=True,
357 auto_create = Bool(True, config=True,
356 help="whether to create the profile_dir if it doesn't exist")
358 help="whether to create the profile_dir if it doesn't exist")
357 classes = List()
359 classes = List()
358 def _classes_default(self,):
360 def _classes_default(self,):
359 from IPython.parallel.apps import launcher
361 from IPython.parallel.apps import launcher
360 return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
362 return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
361
363
362 clean_logs = Bool(True, config=True,
364 clean_logs = Bool(True, config=True,
363 help="whether to cleanup old logs before starting")
365 help="whether to cleanup old logs before starting")
364
366
365 delay = CFloat(1., config=True,
367 delay = CFloat(1., config=True,
366 help="delay (in s) between starting the controller and the engines")
368 help="delay (in s) between starting the controller and the engines")
367
369
368 controller_launcher_class = DottedObjectName('LocalControllerLauncher',
370 controller_launcher_class = DottedObjectName('LocalControllerLauncher',
369 config=True,
371 config=True,
370 help="The class for launching a Controller."
372 help="The class for launching a Controller."
371 )
373 )
372 reset = Bool(False, config=True,
374 reset = Bool(False, config=True,
373 help="Whether to reset config files as part of '--create'."
375 help="Whether to reset config files as part of '--create'."
374 )
376 )
375
377
376 # flags = Dict(flags)
378 # flags = Dict(flags)
377 aliases = Dict(start_aliases)
379 aliases = Dict(start_aliases)
378
380
379 def init_launchers(self):
381 def init_launchers(self):
380 self.controller_launcher = self.build_launcher(self.controller_launcher_class)
382 self.controller_launcher = self.build_launcher(self.controller_launcher_class)
381 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
383 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
382 self.controller_launcher.on_stop(self.stop_launchers)
384 self.controller_launcher.on_stop(self.stop_launchers)
383
385
384 def start_controller(self):
386 def start_controller(self):
385 self.controller_launcher.start(
387 self.controller_launcher.start(
386 self.profile_dir.location
388 self.profile_dir.location
387 )
389 )
388
390
389 def stop_controller(self):
391 def stop_controller(self):
390 # self.log.info("In stop_controller")
392 # self.log.info("In stop_controller")
391 if self.controller_launcher and self.controller_launcher.running:
393 if self.controller_launcher and self.controller_launcher.running:
392 return self.controller_launcher.stop()
394 return self.controller_launcher.stop()
393
395
394 def stop_launchers(self, r=None):
396 def stop_launchers(self, r=None):
395 if not self._stopping:
397 if not self._stopping:
396 self.stop_controller()
398 self.stop_controller()
397 super(IPClusterStart, self).stop_launchers()
399 super(IPClusterStart, self).stop_launchers()
398
400
399 def start(self):
401 def start(self):
400 """Start the app for the start subcommand."""
402 """Start the app for the start subcommand."""
401 # First see if the cluster is already running
403 # First see if the cluster is already running
402 try:
404 try:
403 pid = self.get_pid_from_file()
405 pid = self.get_pid_from_file()
404 except PIDFileError:
406 except PIDFileError:
405 pass
407 pass
406 else:
408 else:
407 if self.check_pid(pid):
409 if self.check_pid(pid):
408 self.log.critical(
410 self.log.critical(
409 'Cluster is already running with [pid=%s]. '
411 'Cluster is already running with [pid=%s]. '
410 'use "ipcluster stop" to stop the cluster.' % pid
412 'use "ipcluster stop" to stop the cluster.' % pid
411 )
413 )
412 # Here I exit with a unusual exit status that other processes
414 # Here I exit with a unusual exit status that other processes
413 # can watch for to learn how I existed.
415 # can watch for to learn how I existed.
414 self.exit(ALREADY_STARTED)
416 self.exit(ALREADY_STARTED)
415 else:
417 else:
416 self.remove_pid_file()
418 self.remove_pid_file()
417
419
418
420
419 # Now log and daemonize
421 # Now log and daemonize
420 self.log.info(
422 self.log.info(
421 'Starting ipcluster with [daemon=%r]' % self.daemonize
423 'Starting ipcluster with [daemon=%r]' % self.daemonize
422 )
424 )
423 # TODO: Get daemonize working on Windows or as a Windows Server.
425 # TODO: Get daemonize working on Windows or as a Windows Server.
424 if self.daemonize:
426 if self.daemonize:
425 if os.name=='posix':
427 if os.name=='posix':
426 daemonize()
428 daemonize()
427
429
428 dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
430 dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
429 dc.start()
431 dc.start()
430 dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
432 dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
431 dc.start()
433 dc.start()
432 # Now write the new pid file AFTER our new forked pid is active.
434 # Now write the new pid file AFTER our new forked pid is active.
433 self.write_pid_file()
435 self.write_pid_file()
434 try:
436 try:
435 self.loop.start()
437 self.loop.start()
436 except KeyboardInterrupt:
438 except KeyboardInterrupt:
437 pass
439 pass
438 except zmq.ZMQError as e:
440 except zmq.ZMQError as e:
439 if e.errno == errno.EINTR:
441 if e.errno == errno.EINTR:
440 pass
442 pass
441 else:
443 else:
442 raise
444 raise
443 finally:
445 finally:
444 self.remove_pid_file()
446 self.remove_pid_file()
445
447
446 base='IPython.parallel.apps.ipclusterapp.IPCluster'
448 base='IPython.parallel.apps.ipclusterapp.IPCluster'
447
449
448 class IPClusterApp(Application):
450 class IPClusterApp(Application):
449 name = u'ipcluster'
451 name = u'ipcluster'
450 description = _description
452 description = _description
451 examples = _main_examples
453 examples = _main_examples
452
454
453 subcommands = {
455 subcommands = {
454 'start' : (base+'Start', start_help),
456 'start' : (base+'Start', start_help),
455 'stop' : (base+'Stop', stop_help),
457 'stop' : (base+'Stop', stop_help),
456 'engines' : (base+'Engines', engines_help),
458 'engines' : (base+'Engines', engines_help),
457 }
459 }
458
460
459 # no aliases or flags for parent App
461 # no aliases or flags for parent App
460 aliases = Dict()
462 aliases = Dict()
461 flags = Dict()
463 flags = Dict()
462
464
463 def start(self):
465 def start(self):
464 if self.subapp is None:
466 if self.subapp is None:
465 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
467 print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
466 print
468 print
467 self.print_description()
469 self.print_description()
468 self.print_subcommands()
470 self.print_subcommands()
469 self.exit(1)
471 self.exit(1)
470 else:
472 else:
471 return self.subapp.start()
473 return self.subapp.start()
472
474
473 def launch_new_instance():
475 def launch_new_instance():
474 """Create and run the IPython cluster."""
476 """Create and run the IPython cluster."""
475 app = IPClusterApp.instance()
477 app = IPClusterApp.instance()
476 app.initialize()
478 app.initialize()
477 app.start()
479 app.start()
478
480
479
481
480 if __name__ == '__main__':
482 if __name__ == '__main__':
481 launch_new_instance()
483 launch_new_instance()
482
484
@@ -1,425 +1,425 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython controller application.
4 The IPython controller application.
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 from __future__ import with_statement
24 from __future__ import with_statement
25
25
26 import os
26 import os
27 import socket
27 import socket
28 import stat
28 import stat
29 import sys
29 import sys
30 import uuid
30 import uuid
31
31
32 from multiprocessing import Process
32 from multiprocessing import Process
33
33
34 import zmq
34 import zmq
35 from zmq.devices import ProcessMonitoredQueue
35 from zmq.devices import ProcessMonitoredQueue
36 from zmq.log.handlers import PUBHandler
36 from zmq.log.handlers import PUBHandler
37 from zmq.utils import jsonapi as json
37 from zmq.utils import jsonapi as json
38
38
39 from IPython.config.application import boolean_flag
39 from IPython.config.application import boolean_flag
40 from IPython.core.profiledir import ProfileDir
40 from IPython.core.profiledir import ProfileDir
41
41
42 from IPython.parallel.apps.baseapp import (
42 from IPython.parallel.apps.baseapp import (
43 BaseParallelApplication,
43 BaseParallelApplication,
44 base_aliases,
44 base_aliases,
45 base_flags,
45 base_flags,
46 )
46 )
47 from IPython.utils.importstring import import_item
47 from IPython.utils.importstring import import_item
48 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict
48 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict
49
49
50 # from IPython.parallel.controller.controller import ControllerFactory
50 # from IPython.parallel.controller.controller import ControllerFactory
51 from IPython.zmq.session import Session
51 from IPython.zmq.session import Session
52 from IPython.parallel.controller.heartmonitor import HeartMonitor
52 from IPython.parallel.controller.heartmonitor import HeartMonitor
53 from IPython.parallel.controller.hub import HubFactory
53 from IPython.parallel.controller.hub import HubFactory
54 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
54 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
55 from IPython.parallel.controller.sqlitedb import SQLiteDB
55 from IPython.parallel.controller.sqlitedb import SQLiteDB
56
56
57 from IPython.parallel.util import signal_children, split_url, asbytes
57 from IPython.parallel.util import signal_children, split_url, asbytes
58
58
59 # conditional import of MongoDB backend class
59 # conditional import of MongoDB backend class
60
60
61 try:
61 try:
62 from IPython.parallel.controller.mongodb import MongoDB
62 from IPython.parallel.controller.mongodb import MongoDB
63 except ImportError:
63 except ImportError:
64 maybe_mongo = []
64 maybe_mongo = []
65 else:
65 else:
66 maybe_mongo = [MongoDB]
66 maybe_mongo = [MongoDB]
67
67
68
68
69 #-----------------------------------------------------------------------------
69 #-----------------------------------------------------------------------------
70 # Module level variables
70 # Module level variables
71 #-----------------------------------------------------------------------------
71 #-----------------------------------------------------------------------------
72
72
73
73
74 #: The default config file name for this application
74 #: The default config file name for this application
75 default_config_file_name = u'ipcontroller_config.py'
75 default_config_file_name = u'ipcontroller_config.py'
76
76
77
77
78 _description = """Start the IPython controller for parallel computing.
78 _description = """Start the IPython controller for parallel computing.
79
79
80 The IPython controller provides a gateway between the IPython engines and
80 The IPython controller provides a gateway between the IPython engines and
81 clients. The controller needs to be started before the engines and can be
81 clients. The controller needs to be started before the engines and can be
82 configured using command line options or using a cluster directory. Cluster
82 configured using command line options or using a cluster directory. Cluster
83 directories contain config, log and security files and are usually located in
83 directories contain config, log and security files and are usually located in
84 your ipython directory and named as "profile_name". See the `profile`
84 your ipython directory and named as "profile_name". See the `profile`
85 and `profile_dir` options for details.
85 and `profile-dir` options for details.
86 """
86 """
87
87
88 _examples = """
88 _examples = """
89 ipcontroller --ip=192.168.0.1 --port=1000 # listen on ip, port for engines
89 ipcontroller --ip=192.168.0.1 --port=1000 # listen on ip, port for engines
90 ipcontroller --scheme=pure # use the pure zeromq scheduler
90 ipcontroller --scheme=pure # use the pure zeromq scheduler
91 """
91 """
92
92
93
93
94 #-----------------------------------------------------------------------------
94 #-----------------------------------------------------------------------------
95 # The main application
95 # The main application
96 #-----------------------------------------------------------------------------
96 #-----------------------------------------------------------------------------
97 flags = {}
97 flags = {}
98 flags.update(base_flags)
98 flags.update(base_flags)
99 flags.update({
99 flags.update({
100 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
100 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
101 'Use threads instead of processes for the schedulers'),
101 'Use threads instead of processes for the schedulers'),
102 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
102 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
103 'use the SQLiteDB backend'),
103 'use the SQLiteDB backend'),
104 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
104 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
105 'use the MongoDB backend'),
105 'use the MongoDB backend'),
106 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
106 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
107 'use the in-memory DictDB backend'),
107 'use the in-memory DictDB backend'),
108 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
108 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
109 'reuse existing json connection files')
109 'reuse existing json connection files')
110 })
110 })
111
111
112 flags.update(boolean_flag('secure', 'IPControllerApp.secure',
112 flags.update(boolean_flag('secure', 'IPControllerApp.secure',
113 "Use HMAC digests for authentication of messages.",
113 "Use HMAC digests for authentication of messages.",
114 "Don't authenticate messages."
114 "Don't authenticate messages."
115 ))
115 ))
116 aliases = dict(
116 aliases = dict(
117 secure = 'IPControllerApp.secure',
117 secure = 'IPControllerApp.secure',
118 ssh = 'IPControllerApp.ssh_server',
118 ssh = 'IPControllerApp.ssh_server',
119 location = 'IPControllerApp.location',
119 location = 'IPControllerApp.location',
120
120
121 ident = 'Session.session',
121 ident = 'Session.session',
122 user = 'Session.username',
122 user = 'Session.username',
123 keyfile = 'Session.keyfile',
123 keyfile = 'Session.keyfile',
124
124
125 url = 'HubFactory.url',
125 url = 'HubFactory.url',
126 ip = 'HubFactory.ip',
126 ip = 'HubFactory.ip',
127 transport = 'HubFactory.transport',
127 transport = 'HubFactory.transport',
128 port = 'HubFactory.regport',
128 port = 'HubFactory.regport',
129
129
130 ping = 'HeartMonitor.period',
130 ping = 'HeartMonitor.period',
131
131
132 scheme = 'TaskScheduler.scheme_name',
132 scheme = 'TaskScheduler.scheme_name',
133 hwm = 'TaskScheduler.hwm',
133 hwm = 'TaskScheduler.hwm',
134 )
134 )
135 aliases.update(base_aliases)
135 aliases.update(base_aliases)
136
136
137
137
138 class IPControllerApp(BaseParallelApplication):
138 class IPControllerApp(BaseParallelApplication):
139
139
140 name = u'ipcontroller'
140 name = u'ipcontroller'
141 description = _description
141 description = _description
142 examples = _examples
142 examples = _examples
143 config_file_name = Unicode(default_config_file_name)
143 config_file_name = Unicode(default_config_file_name)
144 classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
144 classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
145
145
146 # change default to True
146 # change default to True
147 auto_create = Bool(True, config=True,
147 auto_create = Bool(True, config=True,
148 help="""Whether to create profile dir if it doesn't exist.""")
148 help="""Whether to create profile dir if it doesn't exist.""")
149
149
150 reuse_files = Bool(False, config=True,
150 reuse_files = Bool(False, config=True,
151 help='Whether to reuse existing json connection files.'
151 help='Whether to reuse existing json connection files.'
152 )
152 )
153 secure = Bool(True, config=True,
153 secure = Bool(True, config=True,
154 help='Whether to use HMAC digests for extra message authentication.'
154 help='Whether to use HMAC digests for extra message authentication.'
155 )
155 )
156 ssh_server = Unicode(u'', config=True,
156 ssh_server = Unicode(u'', config=True,
157 help="""ssh url for clients to use when connecting to the Controller
157 help="""ssh url for clients to use when connecting to the Controller
158 processes. It should be of the form: [user@]server[:port]. The
158 processes. It should be of the form: [user@]server[:port]. The
159 Controller's listening addresses must be accessible from the ssh server""",
159 Controller's listening addresses must be accessible from the ssh server""",
160 )
160 )
161 location = Unicode(u'', config=True,
161 location = Unicode(u'', config=True,
162 help="""The external IP or domain name of the Controller, used for disambiguating
162 help="""The external IP or domain name of the Controller, used for disambiguating
163 engine and client connections.""",
163 engine and client connections.""",
164 )
164 )
165 import_statements = List([], config=True,
165 import_statements = List([], config=True,
166 help="import statements to be run at startup. Necessary in some environments"
166 help="import statements to be run at startup. Necessary in some environments"
167 )
167 )
168
168
169 use_threads = Bool(False, config=True,
169 use_threads = Bool(False, config=True,
170 help='Use threads instead of processes for the schedulers',
170 help='Use threads instead of processes for the schedulers',
171 )
171 )
172
172
173 # internal
173 # internal
174 children = List()
174 children = List()
175 mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
175 mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
176
176
177 def _use_threads_changed(self, name, old, new):
177 def _use_threads_changed(self, name, old, new):
178 self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
178 self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
179
179
180 aliases = Dict(aliases)
180 aliases = Dict(aliases)
181 flags = Dict(flags)
181 flags = Dict(flags)
182
182
183
183
184 def save_connection_dict(self, fname, cdict):
184 def save_connection_dict(self, fname, cdict):
185 """save a connection dict to json file."""
185 """save a connection dict to json file."""
186 c = self.config
186 c = self.config
187 url = cdict['url']
187 url = cdict['url']
188 location = cdict['location']
188 location = cdict['location']
189 if not location:
189 if not location:
190 try:
190 try:
191 proto,ip,port = split_url(url)
191 proto,ip,port = split_url(url)
192 except AssertionError:
192 except AssertionError:
193 pass
193 pass
194 else:
194 else:
195 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
195 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
196 cdict['location'] = location
196 cdict['location'] = location
197 fname = os.path.join(self.profile_dir.security_dir, fname)
197 fname = os.path.join(self.profile_dir.security_dir, fname)
198 with open(fname, 'wb') as f:
198 with open(fname, 'wb') as f:
199 f.write(json.dumps(cdict, indent=2))
199 f.write(json.dumps(cdict, indent=2))
200 os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
200 os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
201
201
202 def load_config_from_json(self):
202 def load_config_from_json(self):
203 """load config from existing json connector files."""
203 """load config from existing json connector files."""
204 c = self.config
204 c = self.config
205 # load from engine config
205 # load from engine config
206 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
206 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
207 cfg = json.loads(f.read())
207 cfg = json.loads(f.read())
208 key = c.Session.key = asbytes(cfg['exec_key'])
208 key = c.Session.key = asbytes(cfg['exec_key'])
209 xport,addr = cfg['url'].split('://')
209 xport,addr = cfg['url'].split('://')
210 c.HubFactory.engine_transport = xport
210 c.HubFactory.engine_transport = xport
211 ip,ports = addr.split(':')
211 ip,ports = addr.split(':')
212 c.HubFactory.engine_ip = ip
212 c.HubFactory.engine_ip = ip
213 c.HubFactory.regport = int(ports)
213 c.HubFactory.regport = int(ports)
214 self.location = cfg['location']
214 self.location = cfg['location']
215 # load client config
215 # load client config
216 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
216 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
217 cfg = json.loads(f.read())
217 cfg = json.loads(f.read())
218 assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
218 assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
219 xport,addr = cfg['url'].split('://')
219 xport,addr = cfg['url'].split('://')
220 c.HubFactory.client_transport = xport
220 c.HubFactory.client_transport = xport
221 ip,ports = addr.split(':')
221 ip,ports = addr.split(':')
222 c.HubFactory.client_ip = ip
222 c.HubFactory.client_ip = ip
223 self.ssh_server = cfg['ssh']
223 self.ssh_server = cfg['ssh']
224 assert int(ports) == c.HubFactory.regport, "regport mismatch"
224 assert int(ports) == c.HubFactory.regport, "regport mismatch"
225
225
226 def init_hub(self):
226 def init_hub(self):
227 c = self.config
227 c = self.config
228
228
229 self.do_import_statements()
229 self.do_import_statements()
230 reusing = self.reuse_files
230 reusing = self.reuse_files
231 if reusing:
231 if reusing:
232 try:
232 try:
233 self.load_config_from_json()
233 self.load_config_from_json()
234 except (AssertionError,IOError):
234 except (AssertionError,IOError):
235 reusing=False
235 reusing=False
236 # check again, because reusing may have failed:
236 # check again, because reusing may have failed:
237 if reusing:
237 if reusing:
238 pass
238 pass
239 elif self.secure:
239 elif self.secure:
240 key = str(uuid.uuid4())
240 key = str(uuid.uuid4())
241 # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
241 # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
242 # with open(keyfile, 'w') as f:
242 # with open(keyfile, 'w') as f:
243 # f.write(key)
243 # f.write(key)
244 # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
244 # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
245 c.Session.key = asbytes(key)
245 c.Session.key = asbytes(key)
246 else:
246 else:
247 key = c.Session.key = b''
247 key = c.Session.key = b''
248
248
249 try:
249 try:
250 self.factory = HubFactory(config=c, log=self.log)
250 self.factory = HubFactory(config=c, log=self.log)
251 # self.start_logging()
251 # self.start_logging()
252 self.factory.init_hub()
252 self.factory.init_hub()
253 except:
253 except:
254 self.log.error("Couldn't construct the Controller", exc_info=True)
254 self.log.error("Couldn't construct the Controller", exc_info=True)
255 self.exit(1)
255 self.exit(1)
256
256
257 if not reusing:
257 if not reusing:
258 # save to new json config files
258 # save to new json config files
259 f = self.factory
259 f = self.factory
260 cdict = {'exec_key' : key,
260 cdict = {'exec_key' : key,
261 'ssh' : self.ssh_server,
261 'ssh' : self.ssh_server,
262 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
262 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
263 'location' : self.location
263 'location' : self.location
264 }
264 }
265 self.save_connection_dict('ipcontroller-client.json', cdict)
265 self.save_connection_dict('ipcontroller-client.json', cdict)
266 edict = cdict
266 edict = cdict
267 edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
267 edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
268 self.save_connection_dict('ipcontroller-engine.json', edict)
268 self.save_connection_dict('ipcontroller-engine.json', edict)
269
269
270 #
270 #
271 def init_schedulers(self):
271 def init_schedulers(self):
272 children = self.children
272 children = self.children
273 mq = import_item(str(self.mq_class))
273 mq = import_item(str(self.mq_class))
274
274
275 hub = self.factory
275 hub = self.factory
276 # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
276 # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
277 # IOPub relay (in a Process)
277 # IOPub relay (in a Process)
278 q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
278 q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
279 q.bind_in(hub.client_info['iopub'])
279 q.bind_in(hub.client_info['iopub'])
280 q.bind_out(hub.engine_info['iopub'])
280 q.bind_out(hub.engine_info['iopub'])
281 q.setsockopt_out(zmq.SUBSCRIBE, b'')
281 q.setsockopt_out(zmq.SUBSCRIBE, b'')
282 q.connect_mon(hub.monitor_url)
282 q.connect_mon(hub.monitor_url)
283 q.daemon=True
283 q.daemon=True
284 children.append(q)
284 children.append(q)
285
285
286 # Multiplexer Queue (in a Process)
286 # Multiplexer Queue (in a Process)
287 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'in', b'out')
287 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'in', b'out')
288 q.bind_in(hub.client_info['mux'])
288 q.bind_in(hub.client_info['mux'])
289 q.setsockopt_in(zmq.IDENTITY, b'mux')
289 q.setsockopt_in(zmq.IDENTITY, b'mux')
290 q.bind_out(hub.engine_info['mux'])
290 q.bind_out(hub.engine_info['mux'])
291 q.connect_mon(hub.monitor_url)
291 q.connect_mon(hub.monitor_url)
292 q.daemon=True
292 q.daemon=True
293 children.append(q)
293 children.append(q)
294
294
295 # Control Queue (in a Process)
295 # Control Queue (in a Process)
296 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'incontrol', b'outcontrol')
296 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'incontrol', b'outcontrol')
297 q.bind_in(hub.client_info['control'])
297 q.bind_in(hub.client_info['control'])
298 q.setsockopt_in(zmq.IDENTITY, b'control')
298 q.setsockopt_in(zmq.IDENTITY, b'control')
299 q.bind_out(hub.engine_info['control'])
299 q.bind_out(hub.engine_info['control'])
300 q.connect_mon(hub.monitor_url)
300 q.connect_mon(hub.monitor_url)
301 q.daemon=True
301 q.daemon=True
302 children.append(q)
302 children.append(q)
303 try:
303 try:
304 scheme = self.config.TaskScheduler.scheme_name
304 scheme = self.config.TaskScheduler.scheme_name
305 except AttributeError:
305 except AttributeError:
306 scheme = TaskScheduler.scheme_name.get_default_value()
306 scheme = TaskScheduler.scheme_name.get_default_value()
307 # Task Queue (in a Process)
307 # Task Queue (in a Process)
308 if scheme == 'pure':
308 if scheme == 'pure':
309 self.log.warn("task::using pure XREQ Task scheduler")
309 self.log.warn("task::using pure XREQ Task scheduler")
310 q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, b'intask', b'outtask')
310 q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, b'intask', b'outtask')
311 # q.setsockopt_out(zmq.HWM, hub.hwm)
311 # q.setsockopt_out(zmq.HWM, hub.hwm)
312 q.bind_in(hub.client_info['task'][1])
312 q.bind_in(hub.client_info['task'][1])
313 q.setsockopt_in(zmq.IDENTITY, b'task')
313 q.setsockopt_in(zmq.IDENTITY, b'task')
314 q.bind_out(hub.engine_info['task'])
314 q.bind_out(hub.engine_info['task'])
315 q.connect_mon(hub.monitor_url)
315 q.connect_mon(hub.monitor_url)
316 q.daemon=True
316 q.daemon=True
317 children.append(q)
317 children.append(q)
318 elif scheme == 'none':
318 elif scheme == 'none':
319 self.log.warn("task::using no Task scheduler")
319 self.log.warn("task::using no Task scheduler")
320
320
321 else:
321 else:
322 self.log.info("task::using Python %s Task scheduler"%scheme)
322 self.log.info("task::using Python %s Task scheduler"%scheme)
323 sargs = (hub.client_info['task'][1], hub.engine_info['task'],
323 sargs = (hub.client_info['task'][1], hub.engine_info['task'],
324 hub.monitor_url, hub.client_info['notification'])
324 hub.monitor_url, hub.client_info['notification'])
325 kwargs = dict(logname='scheduler', loglevel=self.log_level,
325 kwargs = dict(logname='scheduler', loglevel=self.log_level,
326 log_url = self.log_url, config=dict(self.config))
326 log_url = self.log_url, config=dict(self.config))
327 if 'Process' in self.mq_class:
327 if 'Process' in self.mq_class:
328 # run the Python scheduler in a Process
328 # run the Python scheduler in a Process
329 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
329 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
330 q.daemon=True
330 q.daemon=True
331 children.append(q)
331 children.append(q)
332 else:
332 else:
333 # single-threaded Controller
333 # single-threaded Controller
334 kwargs['in_thread'] = True
334 kwargs['in_thread'] = True
335 launch_scheduler(*sargs, **kwargs)
335 launch_scheduler(*sargs, **kwargs)
336
336
337
337
338 def save_urls(self):
338 def save_urls(self):
339 """save the registration urls to files."""
339 """save the registration urls to files."""
340 c = self.config
340 c = self.config
341
341
342 sec_dir = self.profile_dir.security_dir
342 sec_dir = self.profile_dir.security_dir
343 cf = self.factory
343 cf = self.factory
344
344
345 with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
345 with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
346 f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
346 f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
347
347
348 with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
348 with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
349 f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
349 f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
350
350
351
351
352 def do_import_statements(self):
352 def do_import_statements(self):
353 statements = self.import_statements
353 statements = self.import_statements
354 for s in statements:
354 for s in statements:
355 try:
355 try:
356 self.log.msg("Executing statement: '%s'" % s)
356 self.log.msg("Executing statement: '%s'" % s)
357 exec s in globals(), locals()
357 exec s in globals(), locals()
358 except:
358 except:
359 self.log.msg("Error running statement: %s" % s)
359 self.log.msg("Error running statement: %s" % s)
360
360
361 def forward_logging(self):
361 def forward_logging(self):
362 if self.log_url:
362 if self.log_url:
363 self.log.info("Forwarding logging to %s"%self.log_url)
363 self.log.info("Forwarding logging to %s"%self.log_url)
364 context = zmq.Context.instance()
364 context = zmq.Context.instance()
365 lsock = context.socket(zmq.PUB)
365 lsock = context.socket(zmq.PUB)
366 lsock.connect(self.log_url)
366 lsock.connect(self.log_url)
367 handler = PUBHandler(lsock)
367 handler = PUBHandler(lsock)
368 self.log.removeHandler(self._log_handler)
368 self.log.removeHandler(self._log_handler)
369 handler.root_topic = 'controller'
369 handler.root_topic = 'controller'
370 handler.setLevel(self.log_level)
370 handler.setLevel(self.log_level)
371 self.log.addHandler(handler)
371 self.log.addHandler(handler)
372 self._log_handler = handler
372 self._log_handler = handler
373 # #
373 # #
374
374
375 def initialize(self, argv=None):
375 def initialize(self, argv=None):
376 super(IPControllerApp, self).initialize(argv)
376 super(IPControllerApp, self).initialize(argv)
377 self.forward_logging()
377 self.forward_logging()
378 self.init_hub()
378 self.init_hub()
379 self.init_schedulers()
379 self.init_schedulers()
380
380
381 def start(self):
381 def start(self):
382 # Start the subprocesses:
382 # Start the subprocesses:
383 self.factory.start()
383 self.factory.start()
384 child_procs = []
384 child_procs = []
385 for child in self.children:
385 for child in self.children:
386 child.start()
386 child.start()
387 if isinstance(child, ProcessMonitoredQueue):
387 if isinstance(child, ProcessMonitoredQueue):
388 child_procs.append(child.launcher)
388 child_procs.append(child.launcher)
389 elif isinstance(child, Process):
389 elif isinstance(child, Process):
390 child_procs.append(child)
390 child_procs.append(child)
391 if child_procs:
391 if child_procs:
392 signal_children(child_procs)
392 signal_children(child_procs)
393
393
394 self.write_pid_file(overwrite=True)
394 self.write_pid_file(overwrite=True)
395
395
396 try:
396 try:
397 self.factory.loop.start()
397 self.factory.loop.start()
398 except KeyboardInterrupt:
398 except KeyboardInterrupt:
399 self.log.critical("Interrupted, Exiting...\n")
399 self.log.critical("Interrupted, Exiting...\n")
400
400
401
401
402
402
403 def launch_new_instance():
403 def launch_new_instance():
404 """Create and run the IPython controller"""
404 """Create and run the IPython controller"""
405 if sys.platform == 'win32':
405 if sys.platform == 'win32':
406 # make sure we don't get called from a multiprocessing subprocess
406 # make sure we don't get called from a multiprocessing subprocess
407 # this can result in infinite Controllers being started on Windows
407 # this can result in infinite Controllers being started on Windows
408 # which doesn't have a proper fork, so multiprocessing is wonky
408 # which doesn't have a proper fork, so multiprocessing is wonky
409
409
410 # this only comes up when IPython has been installed using vanilla
410 # this only comes up when IPython has been installed using vanilla
411 # setuptools, and *not* distribute.
411 # setuptools, and *not* distribute.
412 import multiprocessing
412 import multiprocessing
413 p = multiprocessing.current_process()
413 p = multiprocessing.current_process()
414 # the main process has name 'MainProcess'
414 # the main process has name 'MainProcess'
415 # subprocesses will have names like 'Process-1'
415 # subprocesses will have names like 'Process-1'
416 if p.name != 'MainProcess':
416 if p.name != 'MainProcess':
417 # we are a subprocess, don't start another Controller!
417 # we are a subprocess, don't start another Controller!
418 return
418 return
419 app = IPControllerApp.instance()
419 app = IPControllerApp.instance()
420 app.initialize()
420 app.initialize()
421 app.start()
421 app.start()
422
422
423
423
424 if __name__ == '__main__':
424 if __name__ == '__main__':
425 launch_new_instance()
425 launch_new_instance()
@@ -1,307 +1,307 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython engine application
4 The IPython engine application
5
5
6 Authors:
6 Authors:
7
7
8 * Brian Granger
8 * Brian Granger
9 * MinRK
9 * MinRK
10
10
11 """
11 """
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Copyright (C) 2008-2011 The IPython Development Team
14 # Copyright (C) 2008-2011 The IPython Development Team
15 #
15 #
16 # Distributed under the terms of the BSD License. The full license is in
16 # Distributed under the terms of the BSD License. The full license is in
17 # the file COPYING, distributed as part of this software.
17 # the file COPYING, distributed as part of this software.
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Imports
21 # Imports
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 import json
24 import json
25 import os
25 import os
26 import sys
26 import sys
27 import time
27 import time
28
28
29 import zmq
29 import zmq
30 from zmq.eventloop import ioloop
30 from zmq.eventloop import ioloop
31
31
32 from IPython.core.profiledir import ProfileDir
32 from IPython.core.profiledir import ProfileDir
33 from IPython.parallel.apps.baseapp import (
33 from IPython.parallel.apps.baseapp import (
34 BaseParallelApplication,
34 BaseParallelApplication,
35 base_aliases,
35 base_aliases,
36 base_flags,
36 base_flags,
37 )
37 )
38 from IPython.zmq.log import EnginePUBHandler
38 from IPython.zmq.log import EnginePUBHandler
39
39
40 from IPython.config.configurable import Configurable
40 from IPython.config.configurable import Configurable
41 from IPython.zmq.session import Session
41 from IPython.zmq.session import Session
42 from IPython.parallel.engine.engine import EngineFactory
42 from IPython.parallel.engine.engine import EngineFactory
43 from IPython.parallel.engine.streamkernel import Kernel
43 from IPython.parallel.engine.streamkernel import Kernel
44 from IPython.parallel.util import disambiguate_url, asbytes
44 from IPython.parallel.util import disambiguate_url, asbytes
45
45
46 from IPython.utils.importstring import import_item
46 from IPython.utils.importstring import import_item
47 from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float
47 from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float
48
48
49
49
50 #-----------------------------------------------------------------------------
50 #-----------------------------------------------------------------------------
51 # Module level variables
51 # Module level variables
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53
53
54 #: The default config file name for this application
54 #: The default config file name for this application
55 default_config_file_name = u'ipengine_config.py'
55 default_config_file_name = u'ipengine_config.py'
56
56
57 _description = """Start an IPython engine for parallel computing.
57 _description = """Start an IPython engine for parallel computing.
58
58
59 IPython engines run in parallel and perform computations on behalf of a client
59 IPython engines run in parallel and perform computations on behalf of a client
60 and controller. A controller needs to be started before the engines. The
60 and controller. A controller needs to be started before the engines. The
61 engine can be configured using command line options or using a cluster
61 engine can be configured using command line options or using a cluster
62 directory. Cluster directories contain config, log and security files and are
62 directory. Cluster directories contain config, log and security files and are
63 usually located in your ipython directory and named as "profile_name".
63 usually located in your ipython directory and named as "profile_name".
64 See the `profile` and `profile_dir` options for details.
64 See the `profile` and `profile-dir` options for details.
65 """
65 """
66
66
67 _examples = """
67 _examples = """
68 ipengine --ip=192.168.0.1 --port=1000 # connect to hub at ip and port
68 ipengine --ip=192.168.0.1 --port=1000 # connect to hub at ip and port
69 ipengine --log-to-file --log_level=DEBUG # log to a file with DEBUG verbosity
69 ipengine --log-to-file --log-level=DEBUG # log to a file with DEBUG verbosity
70 """
70 """
71
71
72 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
73 # MPI configuration
73 # MPI configuration
74 #-----------------------------------------------------------------------------
74 #-----------------------------------------------------------------------------
75
75
76 mpi4py_init = """from mpi4py import MPI as mpi
76 mpi4py_init = """from mpi4py import MPI as mpi
77 mpi.size = mpi.COMM_WORLD.Get_size()
77 mpi.size = mpi.COMM_WORLD.Get_size()
78 mpi.rank = mpi.COMM_WORLD.Get_rank()
78 mpi.rank = mpi.COMM_WORLD.Get_rank()
79 """
79 """
80
80
81
81
82 pytrilinos_init = """from PyTrilinos import Epetra
82 pytrilinos_init = """from PyTrilinos import Epetra
83 class SimpleStruct:
83 class SimpleStruct:
84 pass
84 pass
85 mpi = SimpleStruct()
85 mpi = SimpleStruct()
86 mpi.rank = 0
86 mpi.rank = 0
87 mpi.size = 0
87 mpi.size = 0
88 """
88 """
89
89
90 class MPI(Configurable):
90 class MPI(Configurable):
91 """Configurable for MPI initialization"""
91 """Configurable for MPI initialization"""
92 use = Unicode('', config=True,
92 use = Unicode('', config=True,
93 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
93 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
94 )
94 )
95
95
96 def _on_use_changed(self, old, new):
96 def _on_use_changed(self, old, new):
97 # load default init script if it's not set
97 # load default init script if it's not set
98 if not self.init_script:
98 if not self.init_script:
99 self.init_script = self.default_inits.get(new, '')
99 self.init_script = self.default_inits.get(new, '')
100
100
101 init_script = Unicode('', config=True,
101 init_script = Unicode('', config=True,
102 help="Initialization code for MPI")
102 help="Initialization code for MPI")
103
103
104 default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
104 default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
105 config=True)
105 config=True)
106
106
107
107
108 #-----------------------------------------------------------------------------
108 #-----------------------------------------------------------------------------
109 # Main application
109 # Main application
110 #-----------------------------------------------------------------------------
110 #-----------------------------------------------------------------------------
111 aliases = dict(
111 aliases = dict(
112 file = 'IPEngineApp.url_file',
112 file = 'IPEngineApp.url_file',
113 c = 'IPEngineApp.startup_command',
113 c = 'IPEngineApp.startup_command',
114 s = 'IPEngineApp.startup_script',
114 s = 'IPEngineApp.startup_script',
115
115
116 ident = 'Session.session',
116 ident = 'Session.session',
117 user = 'Session.username',
117 user = 'Session.username',
118 keyfile = 'Session.keyfile',
118 keyfile = 'Session.keyfile',
119
119
120 url = 'EngineFactory.url',
120 url = 'EngineFactory.url',
121 ip = 'EngineFactory.ip',
121 ip = 'EngineFactory.ip',
122 transport = 'EngineFactory.transport',
122 transport = 'EngineFactory.transport',
123 port = 'EngineFactory.regport',
123 port = 'EngineFactory.regport',
124 location = 'EngineFactory.location',
124 location = 'EngineFactory.location',
125
125
126 timeout = 'EngineFactory.timeout',
126 timeout = 'EngineFactory.timeout',
127
127
128 mpi = 'MPI.use',
128 mpi = 'MPI.use',
129
129
130 )
130 )
131 aliases.update(base_aliases)
131 aliases.update(base_aliases)
132
132
133
133
134 class IPEngineApp(BaseParallelApplication):
134 class IPEngineApp(BaseParallelApplication):
135
135
136 name = Unicode(u'ipengine')
136 name = Unicode(u'ipengine')
137 description = Unicode(_description)
137 description = Unicode(_description)
138 examples = _examples
138 examples = _examples
139 config_file_name = Unicode(default_config_file_name)
139 config_file_name = Unicode(default_config_file_name)
140 classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
140 classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
141
141
142 startup_script = Unicode(u'', config=True,
142 startup_script = Unicode(u'', config=True,
143 help='specify a script to be run at startup')
143 help='specify a script to be run at startup')
144 startup_command = Unicode('', config=True,
144 startup_command = Unicode('', config=True,
145 help='specify a command to be run at startup')
145 help='specify a command to be run at startup')
146
146
147 url_file = Unicode(u'', config=True,
147 url_file = Unicode(u'', config=True,
148 help="""The full location of the file containing the connection information for
148 help="""The full location of the file containing the connection information for
149 the controller. If this is not given, the file must be in the
149 the controller. If this is not given, the file must be in the
150 security directory of the cluster directory. This location is
150 security directory of the cluster directory. This location is
151 resolved using the `profile` or `profile_dir` options.""",
151 resolved using the `profile` or `profile_dir` options.""",
152 )
152 )
153 wait_for_url_file = Float(5, config=True,
153 wait_for_url_file = Float(5, config=True,
154 help="""The maximum number of seconds to wait for url_file to exist.
154 help="""The maximum number of seconds to wait for url_file to exist.
155 This is useful for batch-systems and shared-filesystems where the
155 This is useful for batch-systems and shared-filesystems where the
156 controller and engine are started at the same time and it
156 controller and engine are started at the same time and it
157 may take a moment for the controller to write the connector files.""")
157 may take a moment for the controller to write the connector files.""")
158
158
159 url_file_name = Unicode(u'ipcontroller-engine.json')
159 url_file_name = Unicode(u'ipcontroller-engine.json')
160 log_url = Unicode('', config=True,
160 log_url = Unicode('', config=True,
161 help="""The URL for the iploggerapp instance, for forwarding
161 help="""The URL for the iploggerapp instance, for forwarding
162 logging to a central location.""")
162 logging to a central location.""")
163
163
164 aliases = Dict(aliases)
164 aliases = Dict(aliases)
165
165
166 # def find_key_file(self):
166 # def find_key_file(self):
167 # """Set the key file.
167 # """Set the key file.
168 #
168 #
169 # Here we don't try to actually see if it exists for is valid as that
169 # Here we don't try to actually see if it exists for is valid as that
170 # is hadled by the connection logic.
170 # is hadled by the connection logic.
171 # """
171 # """
172 # config = self.master_config
172 # config = self.master_config
173 # # Find the actual controller key file
173 # # Find the actual controller key file
174 # if not config.Global.key_file:
174 # if not config.Global.key_file:
175 # try_this = os.path.join(
175 # try_this = os.path.join(
176 # config.Global.profile_dir,
176 # config.Global.profile_dir,
177 # config.Global.security_dir,
177 # config.Global.security_dir,
178 # config.Global.key_file_name
178 # config.Global.key_file_name
179 # )
179 # )
180 # config.Global.key_file = try_this
180 # config.Global.key_file = try_this
181
181
182 def find_url_file(self):
182 def find_url_file(self):
183 """Set the url file.
183 """Set the url file.
184
184
185 Here we don't try to actually see if it exists for is valid as that
185 Here we don't try to actually see if it exists for is valid as that
186 is hadled by the connection logic.
186 is hadled by the connection logic.
187 """
187 """
188 config = self.config
188 config = self.config
189 # Find the actual controller key file
189 # Find the actual controller key file
190 if not self.url_file:
190 if not self.url_file:
191 self.url_file = os.path.join(
191 self.url_file = os.path.join(
192 self.profile_dir.security_dir,
192 self.profile_dir.security_dir,
193 self.url_file_name
193 self.url_file_name
194 )
194 )
195 def init_engine(self):
195 def init_engine(self):
196 # This is the working dir by now.
196 # This is the working dir by now.
197 sys.path.insert(0, '')
197 sys.path.insert(0, '')
198 config = self.config
198 config = self.config
199 # print config
199 # print config
200 self.find_url_file()
200 self.find_url_file()
201
201
202 # was the url manually specified?
202 # was the url manually specified?
203 keys = set(self.config.EngineFactory.keys())
203 keys = set(self.config.EngineFactory.keys())
204 keys = keys.union(set(self.config.RegistrationFactory.keys()))
204 keys = keys.union(set(self.config.RegistrationFactory.keys()))
205
205
206 if keys.intersection(set(['ip', 'url', 'port'])):
206 if keys.intersection(set(['ip', 'url', 'port'])):
207 # Connection info was specified, don't wait for the file
207 # Connection info was specified, don't wait for the file
208 url_specified = True
208 url_specified = True
209 self.wait_for_url_file = 0
209 self.wait_for_url_file = 0
210 else:
210 else:
211 url_specified = False
211 url_specified = False
212
212
213 if self.wait_for_url_file and not os.path.exists(self.url_file):
213 if self.wait_for_url_file and not os.path.exists(self.url_file):
214 self.log.warn("url_file %r not found"%self.url_file)
214 self.log.warn("url_file %r not found"%self.url_file)
215 self.log.warn("Waiting up to %.1f seconds for it to arrive."%self.wait_for_url_file)
215 self.log.warn("Waiting up to %.1f seconds for it to arrive."%self.wait_for_url_file)
216 tic = time.time()
216 tic = time.time()
217 while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
217 while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
218 # wait for url_file to exist, for up to 10 seconds
218 # wait for url_file to exist, for up to 10 seconds
219 time.sleep(0.1)
219 time.sleep(0.1)
220
220
221 if os.path.exists(self.url_file):
221 if os.path.exists(self.url_file):
222 self.log.info("Loading url_file %r"%self.url_file)
222 self.log.info("Loading url_file %r"%self.url_file)
223 with open(self.url_file) as f:
223 with open(self.url_file) as f:
224 d = json.loads(f.read())
224 d = json.loads(f.read())
225 if d['exec_key']:
225 if d['exec_key']:
226 config.Session.key = asbytes(d['exec_key'])
226 config.Session.key = asbytes(d['exec_key'])
227 d['url'] = disambiguate_url(d['url'], d['location'])
227 d['url'] = disambiguate_url(d['url'], d['location'])
228 config.EngineFactory.url = d['url']
228 config.EngineFactory.url = d['url']
229 config.EngineFactory.location = d['location']
229 config.EngineFactory.location = d['location']
230 elif not url_specified:
230 elif not url_specified:
231 self.log.critical("Fatal: url file never arrived: %s"%self.url_file)
231 self.log.critical("Fatal: url file never arrived: %s"%self.url_file)
232 self.exit(1)
232 self.exit(1)
233
233
234
234
235 try:
235 try:
236 exec_lines = config.Kernel.exec_lines
236 exec_lines = config.Kernel.exec_lines
237 except AttributeError:
237 except AttributeError:
238 config.Kernel.exec_lines = []
238 config.Kernel.exec_lines = []
239 exec_lines = config.Kernel.exec_lines
239 exec_lines = config.Kernel.exec_lines
240
240
241 if self.startup_script:
241 if self.startup_script:
242 enc = sys.getfilesystemencoding() or 'utf8'
242 enc = sys.getfilesystemencoding() or 'utf8'
243 cmd="execfile(%r)"%self.startup_script.encode(enc)
243 cmd="execfile(%r)"%self.startup_script.encode(enc)
244 exec_lines.append(cmd)
244 exec_lines.append(cmd)
245 if self.startup_command:
245 if self.startup_command:
246 exec_lines.append(self.startup_command)
246 exec_lines.append(self.startup_command)
247
247
248 # Create the underlying shell class and Engine
248 # Create the underlying shell class and Engine
249 # shell_class = import_item(self.master_config.Global.shell_class)
249 # shell_class = import_item(self.master_config.Global.shell_class)
250 # print self.config
250 # print self.config
251 try:
251 try:
252 self.engine = EngineFactory(config=config, log=self.log)
252 self.engine = EngineFactory(config=config, log=self.log)
253 except:
253 except:
254 self.log.error("Couldn't start the Engine", exc_info=True)
254 self.log.error("Couldn't start the Engine", exc_info=True)
255 self.exit(1)
255 self.exit(1)
256
256
257 def forward_logging(self):
257 def forward_logging(self):
258 if self.log_url:
258 if self.log_url:
259 self.log.info("Forwarding logging to %s"%self.log_url)
259 self.log.info("Forwarding logging to %s"%self.log_url)
260 context = self.engine.context
260 context = self.engine.context
261 lsock = context.socket(zmq.PUB)
261 lsock = context.socket(zmq.PUB)
262 lsock.connect(self.log_url)
262 lsock.connect(self.log_url)
263 self.log.removeHandler(self._log_handler)
263 self.log.removeHandler(self._log_handler)
264 handler = EnginePUBHandler(self.engine, lsock)
264 handler = EnginePUBHandler(self.engine, lsock)
265 handler.setLevel(self.log_level)
265 handler.setLevel(self.log_level)
266 self.log.addHandler(handler)
266 self.log.addHandler(handler)
267 self._log_handler = handler
267 self._log_handler = handler
268 #
268 #
269 def init_mpi(self):
269 def init_mpi(self):
270 global mpi
270 global mpi
271 self.mpi = MPI(config=self.config)
271 self.mpi = MPI(config=self.config)
272
272
273 mpi_import_statement = self.mpi.init_script
273 mpi_import_statement = self.mpi.init_script
274 if mpi_import_statement:
274 if mpi_import_statement:
275 try:
275 try:
276 self.log.info("Initializing MPI:")
276 self.log.info("Initializing MPI:")
277 self.log.info(mpi_import_statement)
277 self.log.info(mpi_import_statement)
278 exec mpi_import_statement in globals()
278 exec mpi_import_statement in globals()
279 except:
279 except:
280 mpi = None
280 mpi = None
281 else:
281 else:
282 mpi = None
282 mpi = None
283
283
284 def initialize(self, argv=None):
284 def initialize(self, argv=None):
285 super(IPEngineApp, self).initialize(argv)
285 super(IPEngineApp, self).initialize(argv)
286 self.init_mpi()
286 self.init_mpi()
287 self.init_engine()
287 self.init_engine()
288 self.forward_logging()
288 self.forward_logging()
289
289
290 def start(self):
290 def start(self):
291 self.engine.start()
291 self.engine.start()
292 try:
292 try:
293 self.engine.loop.start()
293 self.engine.loop.start()
294 except KeyboardInterrupt:
294 except KeyboardInterrupt:
295 self.log.critical("Engine Interrupted, shutting down...\n")
295 self.log.critical("Engine Interrupted, shutting down...\n")
296
296
297
297
298 def launch_new_instance():
298 def launch_new_instance():
299 """Create and run the IPython engine"""
299 """Create and run the IPython engine"""
300 app = IPEngineApp.instance()
300 app = IPEngineApp.instance()
301 app.initialize()
301 app.initialize()
302 app.start()
302 app.start()
303
303
304
304
305 if __name__ == '__main__':
305 if __name__ == '__main__':
306 launch_new_instance()
306 launch_new_instance()
307
307
@@ -1,101 +1,101 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 A simple IPython logger application
4 A simple IPython logger application
5
5
6 Authors:
6 Authors:
7
7
8 * MinRK
8 * MinRK
9
9
10 """
10 """
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Copyright (C) 2011 The IPython Development Team
13 # Copyright (C) 2011 The IPython Development Team
14 #
14 #
15 # Distributed under the terms of the BSD License. The full license is in
15 # Distributed under the terms of the BSD License. The full license is in
16 # the file COPYING, distributed as part of this software.
16 # the file COPYING, distributed as part of this software.
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20 # Imports
20 # Imports
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22
22
23 import os
23 import os
24 import sys
24 import sys
25
25
26 import zmq
26 import zmq
27
27
28 from IPython.core.profiledir import ProfileDir
28 from IPython.core.profiledir import ProfileDir
29 from IPython.utils.traitlets import Bool, Dict, Unicode
29 from IPython.utils.traitlets import Bool, Dict, Unicode
30
30
31 from IPython.parallel.apps.baseapp import (
31 from IPython.parallel.apps.baseapp import (
32 BaseParallelApplication,
32 BaseParallelApplication,
33 base_aliases
33 base_aliases
34 )
34 )
35 from IPython.parallel.apps.logwatcher import LogWatcher
35 from IPython.parallel.apps.logwatcher import LogWatcher
36
36
37 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
38 # Module level variables
38 # Module level variables
39 #-----------------------------------------------------------------------------
39 #-----------------------------------------------------------------------------
40
40
41 #: The default config file name for this application
41 #: The default config file name for this application
42 default_config_file_name = u'iplogger_config.py'
42 default_config_file_name = u'iplogger_config.py'
43
43
44 _description = """Start an IPython logger for parallel computing.
44 _description = """Start an IPython logger for parallel computing.
45
45
46 IPython controllers and engines (and your own processes) can broadcast log messages
46 IPython controllers and engines (and your own processes) can broadcast log messages
47 by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
47 by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
48 logger can be configured using command line options or using a cluster
48 logger can be configured using command line options or using a cluster
49 directory. Cluster directories contain config, log and security files and are
49 directory. Cluster directories contain config, log and security files and are
50 usually located in your ipython directory and named as "profile_name".
50 usually located in your ipython directory and named as "profile_name".
51 See the `profile` and `profile_dir` options for details.
51 See the `profile` and `profile-dir` options for details.
52 """
52 """
53
53
54
54
55 #-----------------------------------------------------------------------------
55 #-----------------------------------------------------------------------------
56 # Main application
56 # Main application
57 #-----------------------------------------------------------------------------
57 #-----------------------------------------------------------------------------
58 aliases = {}
58 aliases = {}
59 aliases.update(base_aliases)
59 aliases.update(base_aliases)
60 aliases.update(dict(url='LogWatcher.url', topics='LogWatcher.topics'))
60 aliases.update(dict(url='LogWatcher.url', topics='LogWatcher.topics'))
61
61
62 class IPLoggerApp(BaseParallelApplication):
62 class IPLoggerApp(BaseParallelApplication):
63
63
64 name = u'iplogger'
64 name = u'iplogger'
65 description = _description
65 description = _description
66 config_file_name = Unicode(default_config_file_name)
66 config_file_name = Unicode(default_config_file_name)
67
67
68 classes = [LogWatcher, ProfileDir]
68 classes = [LogWatcher, ProfileDir]
69 aliases = Dict(aliases)
69 aliases = Dict(aliases)
70
70
71 def initialize(self, argv=None):
71 def initialize(self, argv=None):
72 super(IPLoggerApp, self).initialize(argv)
72 super(IPLoggerApp, self).initialize(argv)
73 self.init_watcher()
73 self.init_watcher()
74
74
75 def init_watcher(self):
75 def init_watcher(self):
76 try:
76 try:
77 self.watcher = LogWatcher(config=self.config, log=self.log)
77 self.watcher = LogWatcher(config=self.config, log=self.log)
78 except:
78 except:
79 self.log.error("Couldn't start the LogWatcher", exc_info=True)
79 self.log.error("Couldn't start the LogWatcher", exc_info=True)
80 self.exit(1)
80 self.exit(1)
81 self.log.info("Listening for log messages on %r"%self.watcher.url)
81 self.log.info("Listening for log messages on %r"%self.watcher.url)
82
82
83
83
84 def start(self):
84 def start(self):
85 self.watcher.start()
85 self.watcher.start()
86 try:
86 try:
87 self.watcher.loop.start()
87 self.watcher.loop.start()
88 except KeyboardInterrupt:
88 except KeyboardInterrupt:
89 self.log.critical("Logging Interrupted, shutting down...\n")
89 self.log.critical("Logging Interrupted, shutting down...\n")
90
90
91
91
92 def launch_new_instance():
92 def launch_new_instance():
93 """Create and run the IPython LogWatcher"""
93 """Create and run the IPython LogWatcher"""
94 app = IPLoggerApp.instance()
94 app = IPLoggerApp.instance()
95 app.initialize()
95 app.initialize()
96 app.start()
96 app.start()
97
97
98
98
99 if __name__ == '__main__':
99 if __name__ == '__main__':
100 launch_new_instance()
100 launch_new_instance()
101
101
@@ -1,320 +1,320 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 Job and task components for writing .xml files that the Windows HPC Server
4 Job and task components for writing .xml files that the Windows HPC Server
5 2008 can use to start jobs.
5 2008 can use to start jobs.
6
6
7 Authors:
7 Authors:
8
8
9 * Brian Granger
9 * Brian Granger
10 * MinRK
10 * MinRK
11
11
12 """
12 """
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Copyright (C) 2008-2011 The IPython Development Team
15 # Copyright (C) 2008-2011 The IPython Development Team
16 #
16 #
17 # Distributed under the terms of the BSD License. The full license is in
17 # Distributed under the terms of the BSD License. The full license is in
18 # the file COPYING, distributed as part of this software.
18 # the file COPYING, distributed as part of this software.
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20
20
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22 # Imports
22 # Imports
23 #-----------------------------------------------------------------------------
23 #-----------------------------------------------------------------------------
24
24
25 import os
25 import os
26 import re
26 import re
27 import uuid
27 import uuid
28
28
29 from xml.etree import ElementTree as ET
29 from xml.etree import ElementTree as ET
30
30
31 from IPython.config.configurable import Configurable
31 from IPython.config.configurable import Configurable
32 from IPython.utils.traitlets import (
32 from IPython.utils.traitlets import (
33 Unicode, Int, List, Instance,
33 Unicode, Int, List, Instance,
34 Enum, Bool
34 Enum, Bool
35 )
35 )
36
36
37 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
38 # Job and Task classes
38 # Job and Task classes
39 #-----------------------------------------------------------------------------
39 #-----------------------------------------------------------------------------
40
40
41
41
42 def as_str(value):
42 def as_str(value):
43 if isinstance(value, str):
43 if isinstance(value, str):
44 return value
44 return value
45 elif isinstance(value, bool):
45 elif isinstance(value, bool):
46 if value:
46 if value:
47 return 'true'
47 return 'true'
48 else:
48 else:
49 return 'false'
49 return 'false'
50 elif isinstance(value, (int, float)):
50 elif isinstance(value, (int, float)):
51 return repr(value)
51 return repr(value)
52 else:
52 else:
53 return value
53 return value
54
54
55
55
56 def indent(elem, level=0):
56 def indent(elem, level=0):
57 i = "\n" + level*" "
57 i = "\n" + level*" "
58 if len(elem):
58 if len(elem):
59 if not elem.text or not elem.text.strip():
59 if not elem.text or not elem.text.strip():
60 elem.text = i + " "
60 elem.text = i + " "
61 if not elem.tail or not elem.tail.strip():
61 if not elem.tail or not elem.tail.strip():
62 elem.tail = i
62 elem.tail = i
63 for elem in elem:
63 for elem in elem:
64 indent(elem, level+1)
64 indent(elem, level+1)
65 if not elem.tail or not elem.tail.strip():
65 if not elem.tail or not elem.tail.strip():
66 elem.tail = i
66 elem.tail = i
67 else:
67 else:
68 if level and (not elem.tail or not elem.tail.strip()):
68 if level and (not elem.tail or not elem.tail.strip()):
69 elem.tail = i
69 elem.tail = i
70
70
71
71
72 def find_username():
72 def find_username():
73 domain = os.environ.get('USERDOMAIN')
73 domain = os.environ.get('USERDOMAIN')
74 username = os.environ.get('USERNAME','')
74 username = os.environ.get('USERNAME','')
75 if domain is None:
75 if domain is None:
76 return username
76 return username
77 else:
77 else:
78 return '%s\\%s' % (domain, username)
78 return '%s\\%s' % (domain, username)
79
79
80
80
81 class WinHPCJob(Configurable):
81 class WinHPCJob(Configurable):
82
82
83 job_id = Unicode('')
83 job_id = Unicode('')
84 job_name = Unicode('MyJob', config=True)
84 job_name = Unicode('MyJob', config=True)
85 min_cores = Int(1, config=True)
85 min_cores = Int(1, config=True)
86 max_cores = Int(1, config=True)
86 max_cores = Int(1, config=True)
87 min_sockets = Int(1, config=True)
87 min_sockets = Int(1, config=True)
88 max_sockets = Int(1, config=True)
88 max_sockets = Int(1, config=True)
89 min_nodes = Int(1, config=True)
89 min_nodes = Int(1, config=True)
90 max_nodes = Int(1, config=True)
90 max_nodes = Int(1, config=True)
91 unit_type = Unicode("Core", config=True)
91 unit_type = Unicode("Core", config=True)
92 auto_calculate_min = Bool(True, config=True)
92 auto_calculate_min = Bool(True, config=True)
93 auto_calculate_max = Bool(True, config=True)
93 auto_calculate_max = Bool(True, config=True)
94 run_until_canceled = Bool(False, config=True)
94 run_until_canceled = Bool(False, config=True)
95 is_exclusive = Bool(False, config=True)
95 is_exclusive = Bool(False, config=True)
96 username = Unicode(find_username(), config=True)
96 username = Unicode(find_username(), config=True)
97 job_type = Unicode('Batch', config=True)
97 job_type = Unicode('Batch', config=True)
98 priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
98 priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
99 default_value='Highest', config=True)
99 default_value='Highest', config=True)
100 requested_nodes = Unicode('', config=True)
100 requested_nodes = Unicode('', config=True)
101 project = Unicode('IPython', config=True)
101 project = Unicode('IPython', config=True)
102 xmlns = Unicode('http://schemas.microsoft.com/HPCS2008/scheduler/')
102 xmlns = Unicode('http://schemas.microsoft.com/HPCS2008/scheduler/')
103 version = Unicode("2.000")
103 version = Unicode("2.000")
104 tasks = List([])
104 tasks = List([])
105
105
106 @property
106 @property
107 def owner(self):
107 def owner(self):
108 return self.username
108 return self.username
109
109
110 def _write_attr(self, root, attr, key):
110 def _write_attr(self, root, attr, key):
111 s = as_str(getattr(self, attr, ''))
111 s = as_str(getattr(self, attr, ''))
112 if s:
112 if s:
113 root.set(key, s)
113 root.set(key, s)
114
114
115 def as_element(self):
115 def as_element(self):
116 # We have to add _A_ type things to get the right order than
116 # We have to add _A_ type things to get the right order than
117 # the MSFT XML parser expects.
117 # the MSFT XML parser expects.
118 root = ET.Element('Job')
118 root = ET.Element('Job')
119 self._write_attr(root, 'version', '_A_Version')
119 self._write_attr(root, 'version', '_A_Version')
120 self._write_attr(root, 'job_name', '_B_Name')
120 self._write_attr(root, 'job_name', '_B_Name')
121 self._write_attr(root, 'unit_type', '_C_UnitType')
121 self._write_attr(root, 'unit_type', '_C_UnitType')
122 self._write_attr(root, 'min_cores', '_D_MinCores')
122 self._write_attr(root, 'min_cores', '_D_MinCores')
123 self._write_attr(root, 'max_cores', '_E_MaxCores')
123 self._write_attr(root, 'max_cores', '_E_MaxCores')
124 self._write_attr(root, 'min_sockets', '_F_MinSockets')
124 self._write_attr(root, 'min_sockets', '_F_MinSockets')
125 self._write_attr(root, 'max_sockets', '_G_MaxSockets')
125 self._write_attr(root, 'max_sockets', '_G_MaxSockets')
126 self._write_attr(root, 'min_nodes', '_H_MinNodes')
126 self._write_attr(root, 'min_nodes', '_H_MinNodes')
127 self._write_attr(root, 'max_nodes', '_I_MaxNodes')
127 self._write_attr(root, 'max_nodes', '_I_MaxNodes')
128 self._write_attr(root, 'run_until_canceled', '_J_RunUntilCanceled')
128 self._write_attr(root, 'run_until_canceled', '_J_RunUntilCanceled')
129 self._write_attr(root, 'is_exclusive', '_K_IsExclusive')
129 self._write_attr(root, 'is_exclusive', '_K_IsExclusive')
130 self._write_attr(root, 'username', '_L_UserName')
130 self._write_attr(root, 'username', '_L_UserName')
131 self._write_attr(root, 'job_type', '_M_JobType')
131 self._write_attr(root, 'job_type', '_M_JobType')
132 self._write_attr(root, 'priority', '_N_Priority')
132 self._write_attr(root, 'priority', '_N_Priority')
133 self._write_attr(root, 'requested_nodes', '_O_RequestedNodes')
133 self._write_attr(root, 'requested_nodes', '_O_RequestedNodes')
134 self._write_attr(root, 'auto_calculate_max', '_P_AutoCalculateMax')
134 self._write_attr(root, 'auto_calculate_max', '_P_AutoCalculateMax')
135 self._write_attr(root, 'auto_calculate_min', '_Q_AutoCalculateMin')
135 self._write_attr(root, 'auto_calculate_min', '_Q_AutoCalculateMin')
136 self._write_attr(root, 'project', '_R_Project')
136 self._write_attr(root, 'project', '_R_Project')
137 self._write_attr(root, 'owner', '_S_Owner')
137 self._write_attr(root, 'owner', '_S_Owner')
138 self._write_attr(root, 'xmlns', '_T_xmlns')
138 self._write_attr(root, 'xmlns', '_T_xmlns')
139 dependencies = ET.SubElement(root, "Dependencies")
139 dependencies = ET.SubElement(root, "Dependencies")
140 etasks = ET.SubElement(root, "Tasks")
140 etasks = ET.SubElement(root, "Tasks")
141 for t in self.tasks:
141 for t in self.tasks:
142 etasks.append(t.as_element())
142 etasks.append(t.as_element())
143 return root
143 return root
144
144
145 def tostring(self):
145 def tostring(self):
146 """Return the string representation of the job description XML."""
146 """Return the string representation of the job description XML."""
147 root = self.as_element()
147 root = self.as_element()
148 indent(root)
148 indent(root)
149 txt = ET.tostring(root, encoding="utf-8")
149 txt = ET.tostring(root, encoding="utf-8")
150 # Now remove the tokens used to order the attributes.
150 # Now remove the tokens used to order the attributes.
151 txt = re.sub(r'_[A-Z]_','',txt)
151 txt = re.sub(r'_[A-Z]_','',txt)
152 txt = '<?xml version="1.0" encoding="utf-8"?>\n' + txt
152 txt = '<?xml version="1.0" encoding="utf-8"?>\n' + txt
153 return txt
153 return txt
154
154
155 def write(self, filename):
155 def write(self, filename):
156 """Write the XML job description to a file."""
156 """Write the XML job description to a file."""
157 txt = self.tostring()
157 txt = self.tostring()
158 with open(filename, 'w') as f:
158 with open(filename, 'w') as f:
159 f.write(txt)
159 f.write(txt)
160
160
161 def add_task(self, task):
161 def add_task(self, task):
162 """Add a task to the job.
162 """Add a task to the job.
163
163
164 Parameters
164 Parameters
165 ----------
165 ----------
166 task : :class:`WinHPCTask`
166 task : :class:`WinHPCTask`
167 The task object to add.
167 The task object to add.
168 """
168 """
169 self.tasks.append(task)
169 self.tasks.append(task)
170
170
171
171
172 class WinHPCTask(Configurable):
172 class WinHPCTask(Configurable):
173
173
174 task_id = Unicode('')
174 task_id = Unicode('')
175 task_name = Unicode('')
175 task_name = Unicode('')
176 version = Unicode("2.000")
176 version = Unicode("2.000")
177 min_cores = Int(1, config=True)
177 min_cores = Int(1, config=True)
178 max_cores = Int(1, config=True)
178 max_cores = Int(1, config=True)
179 min_sockets = Int(1, config=True)
179 min_sockets = Int(1, config=True)
180 max_sockets = Int(1, config=True)
180 max_sockets = Int(1, config=True)
181 min_nodes = Int(1, config=True)
181 min_nodes = Int(1, config=True)
182 max_nodes = Int(1, config=True)
182 max_nodes = Int(1, config=True)
183 unit_type = Unicode("Core", config=True)
183 unit_type = Unicode("Core", config=True)
184 command_line = Unicode('', config=True)
184 command_line = Unicode('', config=True)
185 work_directory = Unicode('', config=True)
185 work_directory = Unicode('', config=True)
186 is_rerunnaable = Bool(True, config=True)
186 is_rerunnaable = Bool(True, config=True)
187 std_out_file_path = Unicode('', config=True)
187 std_out_file_path = Unicode('', config=True)
188 std_err_file_path = Unicode('', config=True)
188 std_err_file_path = Unicode('', config=True)
189 is_parametric = Bool(False, config=True)
189 is_parametric = Bool(False, config=True)
190 environment_variables = Instance(dict, args=(), config=True)
190 environment_variables = Instance(dict, args=(), config=True)
191
191
192 def _write_attr(self, root, attr, key):
192 def _write_attr(self, root, attr, key):
193 s = as_str(getattr(self, attr, ''))
193 s = as_str(getattr(self, attr, ''))
194 if s:
194 if s:
195 root.set(key, s)
195 root.set(key, s)
196
196
197 def as_element(self):
197 def as_element(self):
198 root = ET.Element('Task')
198 root = ET.Element('Task')
199 self._write_attr(root, 'version', '_A_Version')
199 self._write_attr(root, 'version', '_A_Version')
200 self._write_attr(root, 'task_name', '_B_Name')
200 self._write_attr(root, 'task_name', '_B_Name')
201 self._write_attr(root, 'min_cores', '_C_MinCores')
201 self._write_attr(root, 'min_cores', '_C_MinCores')
202 self._write_attr(root, 'max_cores', '_D_MaxCores')
202 self._write_attr(root, 'max_cores', '_D_MaxCores')
203 self._write_attr(root, 'min_sockets', '_E_MinSockets')
203 self._write_attr(root, 'min_sockets', '_E_MinSockets')
204 self._write_attr(root, 'max_sockets', '_F_MaxSockets')
204 self._write_attr(root, 'max_sockets', '_F_MaxSockets')
205 self._write_attr(root, 'min_nodes', '_G_MinNodes')
205 self._write_attr(root, 'min_nodes', '_G_MinNodes')
206 self._write_attr(root, 'max_nodes', '_H_MaxNodes')
206 self._write_attr(root, 'max_nodes', '_H_MaxNodes')
207 self._write_attr(root, 'command_line', '_I_CommandLine')
207 self._write_attr(root, 'command_line', '_I_CommandLine')
208 self._write_attr(root, 'work_directory', '_J_WorkDirectory')
208 self._write_attr(root, 'work_directory', '_J_WorkDirectory')
209 self._write_attr(root, 'is_rerunnaable', '_K_IsRerunnable')
209 self._write_attr(root, 'is_rerunnaable', '_K_IsRerunnable')
210 self._write_attr(root, 'std_out_file_path', '_L_StdOutFilePath')
210 self._write_attr(root, 'std_out_file_path', '_L_StdOutFilePath')
211 self._write_attr(root, 'std_err_file_path', '_M_StdErrFilePath')
211 self._write_attr(root, 'std_err_file_path', '_M_StdErrFilePath')
212 self._write_attr(root, 'is_parametric', '_N_IsParametric')
212 self._write_attr(root, 'is_parametric', '_N_IsParametric')
213 self._write_attr(root, 'unit_type', '_O_UnitType')
213 self._write_attr(root, 'unit_type', '_O_UnitType')
214 root.append(self.get_env_vars())
214 root.append(self.get_env_vars())
215 return root
215 return root
216
216
217 def get_env_vars(self):
217 def get_env_vars(self):
218 env_vars = ET.Element('EnvironmentVariables')
218 env_vars = ET.Element('EnvironmentVariables')
219 for k, v in self.environment_variables.iteritems():
219 for k, v in self.environment_variables.iteritems():
220 variable = ET.SubElement(env_vars, "Variable")
220 variable = ET.SubElement(env_vars, "Variable")
221 name = ET.SubElement(variable, "Name")
221 name = ET.SubElement(variable, "Name")
222 name.text = k
222 name.text = k
223 value = ET.SubElement(variable, "Value")
223 value = ET.SubElement(variable, "Value")
224 value.text = v
224 value.text = v
225 return env_vars
225 return env_vars
226
226
227
227
228
228
229 # By declaring these, we can configure the controller and engine separately!
229 # By declaring these, we can configure the controller and engine separately!
230
230
231 class IPControllerJob(WinHPCJob):
231 class IPControllerJob(WinHPCJob):
232 job_name = Unicode('IPController', config=False)
232 job_name = Unicode('IPController', config=False)
233 is_exclusive = Bool(False, config=True)
233 is_exclusive = Bool(False, config=True)
234 username = Unicode(find_username(), config=True)
234 username = Unicode(find_username(), config=True)
235 priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
235 priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
236 default_value='Highest', config=True)
236 default_value='Highest', config=True)
237 requested_nodes = Unicode('', config=True)
237 requested_nodes = Unicode('', config=True)
238 project = Unicode('IPython', config=True)
238 project = Unicode('IPython', config=True)
239
239
240
240
241 class IPEngineSetJob(WinHPCJob):
241 class IPEngineSetJob(WinHPCJob):
242 job_name = Unicode('IPEngineSet', config=False)
242 job_name = Unicode('IPEngineSet', config=False)
243 is_exclusive = Bool(False, config=True)
243 is_exclusive = Bool(False, config=True)
244 username = Unicode(find_username(), config=True)
244 username = Unicode(find_username(), config=True)
245 priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
245 priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
246 default_value='Highest', config=True)
246 default_value='Highest', config=True)
247 requested_nodes = Unicode('', config=True)
247 requested_nodes = Unicode('', config=True)
248 project = Unicode('IPython', config=True)
248 project = Unicode('IPython', config=True)
249
249
250
250
251 class IPControllerTask(WinHPCTask):
251 class IPControllerTask(WinHPCTask):
252
252
253 task_name = Unicode('IPController', config=True)
253 task_name = Unicode('IPController', config=True)
254 controller_cmd = List(['ipcontroller.exe'], config=True)
254 controller_cmd = List(['ipcontroller.exe'], config=True)
255 controller_args = List(['--log-to-file', 'log-level=40'], config=True)
255 controller_args = List(['--log-to-file', '--log-level=40'], config=True)
256 # I don't want these to be configurable
256 # I don't want these to be configurable
257 std_out_file_path = Unicode('', config=False)
257 std_out_file_path = Unicode('', config=False)
258 std_err_file_path = Unicode('', config=False)
258 std_err_file_path = Unicode('', config=False)
259 min_cores = Int(1, config=False)
259 min_cores = Int(1, config=False)
260 max_cores = Int(1, config=False)
260 max_cores = Int(1, config=False)
261 min_sockets = Int(1, config=False)
261 min_sockets = Int(1, config=False)
262 max_sockets = Int(1, config=False)
262 max_sockets = Int(1, config=False)
263 min_nodes = Int(1, config=False)
263 min_nodes = Int(1, config=False)
264 max_nodes = Int(1, config=False)
264 max_nodes = Int(1, config=False)
265 unit_type = Unicode("Core", config=False)
265 unit_type = Unicode("Core", config=False)
266 work_directory = Unicode('', config=False)
266 work_directory = Unicode('', config=False)
267
267
268 def __init__(self, config=None):
268 def __init__(self, config=None):
269 super(IPControllerTask, self).__init__(config=config)
269 super(IPControllerTask, self).__init__(config=config)
270 the_uuid = uuid.uuid1()
270 the_uuid = uuid.uuid1()
271 self.std_out_file_path = os.path.join('log','ipcontroller-%s.out' % the_uuid)
271 self.std_out_file_path = os.path.join('log','ipcontroller-%s.out' % the_uuid)
272 self.std_err_file_path = os.path.join('log','ipcontroller-%s.err' % the_uuid)
272 self.std_err_file_path = os.path.join('log','ipcontroller-%s.err' % the_uuid)
273
273
274 @property
274 @property
275 def command_line(self):
275 def command_line(self):
276 return ' '.join(self.controller_cmd + self.controller_args)
276 return ' '.join(self.controller_cmd + self.controller_args)
277
277
278
278
279 class IPEngineTask(WinHPCTask):
279 class IPEngineTask(WinHPCTask):
280
280
281 task_name = Unicode('IPEngine', config=True)
281 task_name = Unicode('IPEngine', config=True)
282 engine_cmd = List(['ipengine.exe'], config=True)
282 engine_cmd = List(['ipengine.exe'], config=True)
283 engine_args = List(['--log-to-file', 'log_level=40'], config=True)
283 engine_args = List(['--log-to-file', '--log-level=40'], config=True)
284 # I don't want these to be configurable
284 # I don't want these to be configurable
285 std_out_file_path = Unicode('', config=False)
285 std_out_file_path = Unicode('', config=False)
286 std_err_file_path = Unicode('', config=False)
286 std_err_file_path = Unicode('', config=False)
287 min_cores = Int(1, config=False)
287 min_cores = Int(1, config=False)
288 max_cores = Int(1, config=False)
288 max_cores = Int(1, config=False)
289 min_sockets = Int(1, config=False)
289 min_sockets = Int(1, config=False)
290 max_sockets = Int(1, config=False)
290 max_sockets = Int(1, config=False)
291 min_nodes = Int(1, config=False)
291 min_nodes = Int(1, config=False)
292 max_nodes = Int(1, config=False)
292 max_nodes = Int(1, config=False)
293 unit_type = Unicode("Core", config=False)
293 unit_type = Unicode("Core", config=False)
294 work_directory = Unicode('', config=False)
294 work_directory = Unicode('', config=False)
295
295
296 def __init__(self, config=None):
296 def __init__(self, config=None):
297 super(IPEngineTask,self).__init__(config=config)
297 super(IPEngineTask,self).__init__(config=config)
298 the_uuid = uuid.uuid1()
298 the_uuid = uuid.uuid1()
299 self.std_out_file_path = os.path.join('log','ipengine-%s.out' % the_uuid)
299 self.std_out_file_path = os.path.join('log','ipengine-%s.out' % the_uuid)
300 self.std_err_file_path = os.path.join('log','ipengine-%s.err' % the_uuid)
300 self.std_err_file_path = os.path.join('log','ipengine-%s.err' % the_uuid)
301
301
302 @property
302 @property
303 def command_line(self):
303 def command_line(self):
304 return ' '.join(self.engine_cmd + self.engine_args)
304 return ' '.join(self.engine_cmd + self.engine_args)
305
305
306
306
307 # j = WinHPCJob(None)
307 # j = WinHPCJob(None)
308 # j.job_name = 'IPCluster'
308 # j.job_name = 'IPCluster'
309 # j.username = 'GNET\\bgranger'
309 # j.username = 'GNET\\bgranger'
310 # j.requested_nodes = 'GREEN'
310 # j.requested_nodes = 'GREEN'
311 #
311 #
312 # t = WinHPCTask(None)
312 # t = WinHPCTask(None)
313 # t.task_name = 'Controller'
313 # t.task_name = 'Controller'
314 # t.command_line = r"\\blue\domainusers$\bgranger\Python\Python25\Scripts\ipcontroller.exe --log-to-file -p default --log-level 10"
314 # t.command_line = r"\\blue\domainusers$\bgranger\Python\Python25\Scripts\ipcontroller.exe --log-to-file -p default --log-level 10"
315 # t.work_directory = r"\\blue\domainusers$\bgranger\.ipython\cluster_default"
315 # t.work_directory = r"\\blue\domainusers$\bgranger\.ipython\cluster_default"
316 # t.std_out_file_path = 'controller-out.txt'
316 # t.std_out_file_path = 'controller-out.txt'
317 # t.std_err_file_path = 'controller-err.txt'
317 # t.std_err_file_path = 'controller-err.txt'
318 # t.environment_variables['PYTHONPATH'] = r"\\blue\domainusers$\bgranger\Python\Python25\Lib\site-packages"
318 # t.environment_variables['PYTHONPATH'] = r"\\blue\domainusers$\bgranger\Python\Python25\Lib\site-packages"
319 # j.add_task(t)
319 # j.add_task(t)
320
320
General Comments 0
You need to be logged in to leave comments. Login now