##// END OF EJS Templates
Fixing command line options and help strings to use new syntax....
Brian E. Granger -
Show More
@@ -1,378 +1,378 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The :class:`~IPython.core.application.Application` object for the command
5 5 line :command:`ipython` program.
6 6
7 7 Authors
8 8 -------
9 9
10 10 * Brian Granger
11 11 * Fernando Perez
12 12 * Min Ragan-Kelley
13 13 """
14 14
15 15 #-----------------------------------------------------------------------------
16 16 # Copyright (C) 2008-2010 The IPython Development Team
17 17 #
18 18 # Distributed under the terms of the BSD License. The full license is in
19 19 # the file COPYING, distributed as part of this software.
20 20 #-----------------------------------------------------------------------------
21 21
22 22 #-----------------------------------------------------------------------------
23 23 # Imports
24 24 #-----------------------------------------------------------------------------
25 25
26 26 from __future__ import absolute_import
27 27
28 28 import logging
29 29 import os
30 30 import sys
31 31
32 32 from IPython.config.loader import (
33 33 Config, PyFileConfigLoader
34 34 )
35 35 from IPython.config.application import boolean_flag
36 36 from IPython.core import release
37 37 from IPython.core import usage
38 38 from IPython.core.crashhandler import CrashHandler
39 39 from IPython.core.formatters import PlainTextFormatter
40 40 from IPython.core.application import (
41 41 ProfileDir, BaseIPythonApplication, base_flags, base_aliases
42 42 )
43 43 from IPython.core.shellapp import (
44 44 InteractiveShellApp, shell_flags, shell_aliases
45 45 )
46 46 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
47 47 from IPython.lib import inputhook
48 48 from IPython.utils import warn
49 49 from IPython.utils.path import get_ipython_dir, check_for_old_config
50 50 from IPython.utils.traitlets import (
51 51 Bool, Dict, CaselessStrEnum
52 52 )
53 53
54 54 #-----------------------------------------------------------------------------
55 55 # Globals, utilities and helpers
56 56 #-----------------------------------------------------------------------------
57 57
58 58 #: The default config file name for this application.
59 59 default_config_file_name = u'ipython_config.py'
60 60
61 61 _examples = """
62 62 ipython --pylab # start in pylab mode
63 63 ipython --pylab=qt # start in pylab mode with the qt4 backend
64 ipython --log_level=DEBUG # set logging to DEBUG
64 ipython --log-level=DEBUG # set logging to DEBUG
65 65 ipython --profile=foo # start with profile foo
66 66
67 67 ipython qtconsole # start the qtconsole GUI application
68 68 ipython qtconsole -h # show the help string for the qtconsole subcmd
69 69
70 70 ipython profile create foo # create profile foo w/ default config files
71 71 ipython profile -h # show the help string for the profile subcmd
72 72 """
73 73
74 74 #-----------------------------------------------------------------------------
75 75 # Crash handler for this application
76 76 #-----------------------------------------------------------------------------
77 77
78 78 class IPAppCrashHandler(CrashHandler):
79 79 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
80 80
81 81 def __init__(self, app):
82 82 contact_name = release.authors['Fernando'][0]
83 83 contact_email = release.authors['Fernando'][1]
84 84 bug_tracker = 'http://github.com/ipython/ipython/issues'
85 85 super(IPAppCrashHandler,self).__init__(
86 86 app, contact_name, contact_email, bug_tracker
87 87 )
88 88
89 89 def make_report(self,traceback):
90 90 """Return a string containing a crash report."""
91 91
92 92 sec_sep = self.section_sep
93 93 # Start with parent report
94 94 report = [super(IPAppCrashHandler, self).make_report(traceback)]
95 95 # Add interactive-specific info we may have
96 96 rpt_add = report.append
97 97 try:
98 98 rpt_add(sec_sep+"History of session input:")
99 99 for line in self.app.shell.user_ns['_ih']:
100 100 rpt_add(line)
101 101 rpt_add('\n*** Last line of input (may not be in above history):\n')
102 102 rpt_add(self.app.shell._last_input_line+'\n')
103 103 except:
104 104 pass
105 105
106 106 return ''.join(report)
107 107
108 108 #-----------------------------------------------------------------------------
109 109 # Aliases and Flags
110 110 #-----------------------------------------------------------------------------
111 111 flags = dict(base_flags)
112 112 flags.update(shell_flags)
113 113 addflag = lambda *args: flags.update(boolean_flag(*args))
114 114 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
115 115 'Turn on auto editing of files with syntax errors.',
116 116 'Turn off auto editing of files with syntax errors.'
117 117 )
118 118 addflag('banner', 'TerminalIPythonApp.display_banner',
119 119 "Display a banner upon starting IPython.",
120 120 "Don't display a banner upon starting IPython."
121 121 )
122 122 addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
123 123 """Set to confirm when you try to exit IPython with an EOF (Control-D
124 124 in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
125 125 you can force a direct exit without any confirmation.""",
126 126 "Don't prompt the user when exiting."
127 127 )
128 128 addflag('term-title', 'TerminalInteractiveShell.term_title',
129 129 "Enable auto setting the terminal title.",
130 130 "Disable auto setting the terminal title."
131 131 )
132 132 classic_config = Config()
133 133 classic_config.InteractiveShell.cache_size = 0
134 134 classic_config.PlainTextFormatter.pprint = False
135 135 classic_config.InteractiveShell.prompt_in1 = '>>> '
136 136 classic_config.InteractiveShell.prompt_in2 = '... '
137 137 classic_config.InteractiveShell.prompt_out = ''
138 138 classic_config.InteractiveShell.separate_in = ''
139 139 classic_config.InteractiveShell.separate_out = ''
140 140 classic_config.InteractiveShell.separate_out2 = ''
141 141 classic_config.InteractiveShell.colors = 'NoColor'
142 142 classic_config.InteractiveShell.xmode = 'Plain'
143 143
144 144 flags['classic']=(
145 145 classic_config,
146 146 "Gives IPython a similar feel to the classic Python prompt."
147 147 )
148 148 # # log doesn't make so much sense this way anymore
149 149 # paa('--log','-l',
150 150 # action='store_true', dest='InteractiveShell.logstart',
151 151 # help="Start logging to the default log file (./ipython_log.py).")
152 152 #
153 153 # # quick is harder to implement
154 154 flags['quick']=(
155 155 {'TerminalIPythonApp' : {'quick' : True}},
156 156 "Enable quick startup with no config files."
157 157 )
158 158
159 159 flags['i'] = (
160 160 {'TerminalIPythonApp' : {'force_interact' : True}},
161 161 """also works as '-i'
162 162 If running code from the command line, become interactive afterwards."""
163 163 )
164 164 flags['pylab'] = (
165 165 {'TerminalIPythonApp' : {'pylab' : 'auto'}},
166 166 """Pre-load matplotlib and numpy for interactive use with
167 167 the default matplotlib backend."""
168 168 )
169 169
170 170 aliases = dict(base_aliases)
171 171 aliases.update(shell_aliases)
172 172
173 173 # it's possible we don't want short aliases for *all* of these:
174 174 aliases.update(dict(
175 175 gui='TerminalIPythonApp.gui',
176 176 pylab='TerminalIPythonApp.pylab',
177 177 ))
178 178
179 179 #-----------------------------------------------------------------------------
180 180 # Main classes and functions
181 181 #-----------------------------------------------------------------------------
182 182
183 183
184 184 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
185 185 name = u'ipython'
186 186 description = usage.cl_usage
187 187 default_config_file_name = default_config_file_name
188 188 crash_handler_class = IPAppCrashHandler
189 189 examples = _examples
190 190
191 191 flags = Dict(flags)
192 192 aliases = Dict(aliases)
193 193 classes = [InteractiveShellApp, TerminalInteractiveShell, ProfileDir, PlainTextFormatter]
194 194 subcommands = Dict(dict(
195 195 qtconsole=('IPython.frontend.qt.console.qtconsoleapp.IPythonQtConsoleApp',
196 196 """Launch the IPython Qt Console."""
197 197 ),
198 198 profile = ("IPython.core.profileapp.ProfileApp",
199 199 "Create and manage IPython profiles.")
200 200 ))
201 201
202 202 # *do* autocreate requested profile, but don't create the config file.
203 203 auto_create=Bool(True)
204 204 # configurables
205 205 ignore_old_config=Bool(False, config=True,
206 206 help="Suppress warning messages about legacy config files"
207 207 )
208 208 quick = Bool(False, config=True,
209 209 help="""Start IPython quickly by skipping the loading of config files."""
210 210 )
211 211 def _quick_changed(self, name, old, new):
212 212 if new:
213 213 self.load_config_file = lambda *a, **kw: None
214 214 self.ignore_old_config=True
215 215
216 216 gui = CaselessStrEnum(('qt','wx','gtk'), config=True,
217 217 help="Enable GUI event loop integration ('qt', 'wx', 'gtk')."
218 218 )
219 219 pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'auto'],
220 220 config=True,
221 221 help="""Pre-load matplotlib and numpy for interactive use,
222 222 selecting a particular matplotlib backend and loop integration.
223 223 """
224 224 )
225 225 display_banner = Bool(True, config=True,
226 226 help="Whether to display a banner upon starting IPython."
227 227 )
228 228
229 229 # if there is code of files to run from the cmd line, don't interact
230 230 # unless the --i flag (App.force_interact) is true.
231 231 force_interact = Bool(False, config=True,
232 232 help="""If a command or file is given via the command-line,
233 233 e.g. 'ipython foo.py"""
234 234 )
235 235 def _force_interact_changed(self, name, old, new):
236 236 if new:
237 237 self.interact = True
238 238
239 239 def _file_to_run_changed(self, name, old, new):
240 240 if new and not self.force_interact:
241 241 self.interact = False
242 242 _code_to_run_changed = _file_to_run_changed
243 243
244 244 # internal, not-configurable
245 245 interact=Bool(True)
246 246
247 247
248 248 def parse_command_line(self, argv=None):
249 249 """override to allow old '-pylab' flag with deprecation warning"""
250 250 argv = sys.argv[1:] if argv is None else argv
251 251
252 252 try:
253 253 idx = argv.index('-pylab')
254 254 except ValueError:
255 255 # `-pylab` not given, proceed as normal
256 256 pass
257 257 else:
258 258 # deprecated `-pylab` given,
259 259 # warn and transform into current syntax
260 260 argv = list(argv) # copy, don't clobber
261 261 warn.warn("`-pylab` flag has been deprecated.\n"
262 262 " Use `--pylab` instead, or `--pylab=foo` to specify a backend.")
263 263 sub = '--pylab'
264 264 if len(argv) > idx+1:
265 265 # check for gui arg, as in '-pylab qt'
266 266 gui = argv[idx+1]
267 267 if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):
268 268 sub = '--pylab='+gui
269 269 argv.pop(idx+1)
270 270 argv[idx] = sub
271 271
272 272 return super(TerminalIPythonApp, self).parse_command_line(argv)
273 273
274 274 def initialize(self, argv=None):
275 275 """Do actions after construct, but before starting the app."""
276 276 super(TerminalIPythonApp, self).initialize(argv)
277 277 if self.subapp is not None:
278 278 # don't bother initializing further, starting subapp
279 279 return
280 280 if not self.ignore_old_config:
281 281 check_for_old_config(self.ipython_dir)
282 282 # print self.extra_args
283 283 if self.extra_args:
284 284 self.file_to_run = self.extra_args[0]
285 285 # create the shell
286 286 self.init_shell()
287 287 # and draw the banner
288 288 self.init_banner()
289 289 # Now a variety of things that happen after the banner is printed.
290 290 self.init_gui_pylab()
291 291 self.init_extensions()
292 292 self.init_code()
293 293
294 294 def init_shell(self):
295 295 """initialize the InteractiveShell instance"""
296 296 # I am a little hesitant to put these into InteractiveShell itself.
297 297 # But that might be the place for them
298 298 sys.path.insert(0, '')
299 299
300 300 # Create an InteractiveShell instance.
301 301 # shell.display_banner should always be False for the terminal
302 302 # based app, because we call shell.show_banner() by hand below
303 303 # so the banner shows *before* all extension loading stuff.
304 304 self.shell = TerminalInteractiveShell.instance(config=self.config,
305 305 display_banner=False, profile_dir=self.profile_dir,
306 306 ipython_dir=self.ipython_dir)
307 307
308 308 def init_banner(self):
309 309 """optionally display the banner"""
310 310 if self.display_banner and self.interact:
311 311 self.shell.show_banner()
312 312 # Make sure there is a space below the banner.
313 313 if self.log_level <= logging.INFO: print
314 314
315 315
316 316 def init_gui_pylab(self):
317 317 """Enable GUI event loop integration, taking pylab into account."""
318 318 gui = self.gui
319 319
320 320 # Using `pylab` will also require gui activation, though which toolkit
321 321 # to use may be chosen automatically based on mpl configuration.
322 322 if self.pylab:
323 323 activate = self.shell.enable_pylab
324 324 if self.pylab == 'auto':
325 325 gui = None
326 326 else:
327 327 gui = self.pylab
328 328 else:
329 329 # Enable only GUI integration, no pylab
330 330 activate = inputhook.enable_gui
331 331
332 332 if gui or self.pylab:
333 333 try:
334 334 self.log.info("Enabling GUI event loop integration, "
335 335 "toolkit=%s, pylab=%s" % (gui, self.pylab) )
336 336 activate(gui)
337 337 except:
338 338 self.log.warn("Error in enabling GUI event loop integration:")
339 339 self.shell.showtraceback()
340 340
341 341 def start(self):
342 342 if self.subapp is not None:
343 343 return self.subapp.start()
344 344 # perform any prexec steps:
345 345 if self.interact:
346 346 self.log.debug("Starting IPython's mainloop...")
347 347 self.shell.mainloop()
348 348 else:
349 349 self.log.debug("IPython not interactive...")
350 350
351 351
352 352 def load_default_config(ipython_dir=None):
353 353 """Load the default config file from the default ipython_dir.
354 354
355 355 This is useful for embedded shells.
356 356 """
357 357 if ipython_dir is None:
358 358 ipython_dir = get_ipython_dir()
359 359 profile_dir = os.path.join(ipython_dir, 'profile_default')
360 360 cl = PyFileConfigLoader(default_config_file_name, profile_dir)
361 361 try:
362 362 config = cl.load_config()
363 363 except IOError:
364 364 # no config found
365 365 config = Config()
366 366 return config
367 367
368 368
369 369 def launch_new_instance():
370 370 """Create and run a full blown IPython instance"""
371 371 app = TerminalIPythonApp.instance()
372 372 app.initialize()
373 373 app.start()
374 374
375 375
376 376 if __name__ == '__main__':
377 377 launch_new_instance()
378 378
@@ -1,1065 +1,1065 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 Facilities for launching IPython processes asynchronously.
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * MinRK
10 10 """
11 11
12 12 #-----------------------------------------------------------------------------
13 13 # Copyright (C) 2008-2011 The IPython Development Team
14 14 #
15 15 # Distributed under the terms of the BSD License. The full license is in
16 16 # the file COPYING, distributed as part of this software.
17 17 #-----------------------------------------------------------------------------
18 18
19 19 #-----------------------------------------------------------------------------
20 20 # Imports
21 21 #-----------------------------------------------------------------------------
22 22
23 23 import copy
24 24 import logging
25 25 import os
26 26 import re
27 27 import stat
28 28
29 29 # signal imports, handling various platforms, versions
30 30
31 31 from signal import SIGINT, SIGTERM
32 32 try:
33 33 from signal import SIGKILL
34 34 except ImportError:
35 35 # Windows
36 36 SIGKILL=SIGTERM
37 37
38 38 try:
39 39 # Windows >= 2.7, 3.2
40 40 from signal import CTRL_C_EVENT as SIGINT
41 41 except ImportError:
42 42 pass
43 43
44 44 from subprocess import Popen, PIPE, STDOUT
45 45 try:
46 46 from subprocess import check_output
47 47 except ImportError:
48 48 # pre-2.7, define check_output with Popen
49 49 def check_output(*args, **kwargs):
50 50 kwargs.update(dict(stdout=PIPE))
51 51 p = Popen(*args, **kwargs)
52 52 out,err = p.communicate()
53 53 return out
54 54
55 55 from zmq.eventloop import ioloop
56 56
57 57 from IPython.config.application import Application
58 58 from IPython.config.configurable import LoggingConfigurable
59 59 from IPython.utils.text import EvalFormatter
60 60 from IPython.utils.traitlets import Any, Int, List, Unicode, Dict, Instance
61 61 from IPython.utils.path import get_ipython_module_path
62 62 from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
63 63
64 64 from .win32support import forward_read_events
65 65
66 66 from .winhpcjob import IPControllerTask, IPEngineTask, IPControllerJob, IPEngineSetJob
67 67
68 68 WINDOWS = os.name == 'nt'
69 69
70 70 #-----------------------------------------------------------------------------
71 71 # Paths to the kernel apps
72 72 #-----------------------------------------------------------------------------
73 73
74 74
75 75 ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
76 76 'IPython.parallel.apps.ipclusterapp'
77 77 ))
78 78
79 79 ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
80 80 'IPython.parallel.apps.ipengineapp'
81 81 ))
82 82
83 83 ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
84 84 'IPython.parallel.apps.ipcontrollerapp'
85 85 ))
86 86
87 87 #-----------------------------------------------------------------------------
88 88 # Base launchers and errors
89 89 #-----------------------------------------------------------------------------
90 90
91 91
92 92 class LauncherError(Exception):
93 93 pass
94 94
95 95
96 96 class ProcessStateError(LauncherError):
97 97 pass
98 98
99 99
100 100 class UnknownStatus(LauncherError):
101 101 pass
102 102
103 103
104 104 class BaseLauncher(LoggingConfigurable):
105 105 """An asbtraction for starting, stopping and signaling a process."""
106 106
107 107 # In all of the launchers, the work_dir is where child processes will be
108 108 # run. This will usually be the profile_dir, but may not be. any work_dir
109 109 # passed into the __init__ method will override the config value.
110 110 # This should not be used to set the work_dir for the actual engine
111 111 # and controller. Instead, use their own config files or the
112 112 # controller_args, engine_args attributes of the launchers to add
113 113 # the work_dir option.
114 114 work_dir = Unicode(u'.')
115 115 loop = Instance('zmq.eventloop.ioloop.IOLoop')
116 116
117 117 start_data = Any()
118 118 stop_data = Any()
119 119
120 120 def _loop_default(self):
121 121 return ioloop.IOLoop.instance()
122 122
123 123 def __init__(self, work_dir=u'.', config=None, **kwargs):
124 124 super(BaseLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs)
125 125 self.state = 'before' # can be before, running, after
126 126 self.stop_callbacks = []
127 127 self.start_data = None
128 128 self.stop_data = None
129 129
130 130 @property
131 131 def args(self):
132 132 """A list of cmd and args that will be used to start the process.
133 133
134 134 This is what is passed to :func:`spawnProcess` and the first element
135 135 will be the process name.
136 136 """
137 137 return self.find_args()
138 138
139 139 def find_args(self):
140 140 """The ``.args`` property calls this to find the args list.
141 141
142 142 Subcommand should implement this to construct the cmd and args.
143 143 """
144 144 raise NotImplementedError('find_args must be implemented in a subclass')
145 145
146 146 @property
147 147 def arg_str(self):
148 148 """The string form of the program arguments."""
149 149 return ' '.join(self.args)
150 150
151 151 @property
152 152 def running(self):
153 153 """Am I running."""
154 154 if self.state == 'running':
155 155 return True
156 156 else:
157 157 return False
158 158
159 159 def start(self):
160 160 """Start the process."""
161 161 raise NotImplementedError('start must be implemented in a subclass')
162 162
163 163 def stop(self):
164 164 """Stop the process and notify observers of stopping.
165 165
166 166 This method will return None immediately.
167 167 To observe the actual process stopping, see :meth:`on_stop`.
168 168 """
169 169 raise NotImplementedError('stop must be implemented in a subclass')
170 170
171 171 def on_stop(self, f):
172 172 """Register a callback to be called with this Launcher's stop_data
173 173 when the process actually finishes.
174 174 """
175 175 if self.state=='after':
176 176 return f(self.stop_data)
177 177 else:
178 178 self.stop_callbacks.append(f)
179 179
180 180 def notify_start(self, data):
181 181 """Call this to trigger startup actions.
182 182
183 183 This logs the process startup and sets the state to 'running'. It is
184 184 a pass-through so it can be used as a callback.
185 185 """
186 186
187 187 self.log.info('Process %r started: %r' % (self.args[0], data))
188 188 self.start_data = data
189 189 self.state = 'running'
190 190 return data
191 191
192 192 def notify_stop(self, data):
193 193 """Call this to trigger process stop actions.
194 194
195 195 This logs the process stopping and sets the state to 'after'. Call
196 196 this to trigger callbacks registered via :meth:`on_stop`."""
197 197
198 198 self.log.info('Process %r stopped: %r' % (self.args[0], data))
199 199 self.stop_data = data
200 200 self.state = 'after'
201 201 for i in range(len(self.stop_callbacks)):
202 202 d = self.stop_callbacks.pop()
203 203 d(data)
204 204 return data
205 205
206 206 def signal(self, sig):
207 207 """Signal the process.
208 208
209 209 Parameters
210 210 ----------
211 211 sig : str or int
212 212 'KILL', 'INT', etc., or any signal number
213 213 """
214 214 raise NotImplementedError('signal must be implemented in a subclass')
215 215
216 216
217 217 #-----------------------------------------------------------------------------
218 218 # Local process launchers
219 219 #-----------------------------------------------------------------------------
220 220
221 221
222 222 class LocalProcessLauncher(BaseLauncher):
223 223 """Start and stop an external process in an asynchronous manner.
224 224
225 225 This will launch the external process with a working directory of
226 226 ``self.work_dir``.
227 227 """
228 228
229 229 # This is used to to construct self.args, which is passed to
230 230 # spawnProcess.
231 231 cmd_and_args = List([])
232 232 poll_frequency = Int(100) # in ms
233 233
234 234 def __init__(self, work_dir=u'.', config=None, **kwargs):
235 235 super(LocalProcessLauncher, self).__init__(
236 236 work_dir=work_dir, config=config, **kwargs
237 237 )
238 238 self.process = None
239 239 self.poller = None
240 240
241 241 def find_args(self):
242 242 return self.cmd_and_args
243 243
244 244 def start(self):
245 245 if self.state == 'before':
246 246 self.process = Popen(self.args,
247 247 stdout=PIPE,stderr=PIPE,stdin=PIPE,
248 248 env=os.environ,
249 249 cwd=self.work_dir
250 250 )
251 251 if WINDOWS:
252 252 self.stdout = forward_read_events(self.process.stdout)
253 253 self.stderr = forward_read_events(self.process.stderr)
254 254 else:
255 255 self.stdout = self.process.stdout.fileno()
256 256 self.stderr = self.process.stderr.fileno()
257 257 self.loop.add_handler(self.stdout, self.handle_stdout, self.loop.READ)
258 258 self.loop.add_handler(self.stderr, self.handle_stderr, self.loop.READ)
259 259 self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop)
260 260 self.poller.start()
261 261 self.notify_start(self.process.pid)
262 262 else:
263 263 s = 'The process was already started and has state: %r' % self.state
264 264 raise ProcessStateError(s)
265 265
266 266 def stop(self):
267 267 return self.interrupt_then_kill()
268 268
269 269 def signal(self, sig):
270 270 if self.state == 'running':
271 271 if WINDOWS and sig != SIGINT:
272 272 # use Windows tree-kill for better child cleanup
273 273 check_output(['taskkill', '-pid', str(self.process.pid), '-t', '-f'])
274 274 else:
275 275 self.process.send_signal(sig)
276 276
277 277 def interrupt_then_kill(self, delay=2.0):
278 278 """Send INT, wait a delay and then send KILL."""
279 279 try:
280 280 self.signal(SIGINT)
281 281 except Exception:
282 282 self.log.debug("interrupt failed")
283 283 pass
284 284 self.killer = ioloop.DelayedCallback(lambda : self.signal(SIGKILL), delay*1000, self.loop)
285 285 self.killer.start()
286 286
287 287 # callbacks, etc:
288 288
289 289 def handle_stdout(self, fd, events):
290 290 if WINDOWS:
291 291 line = self.stdout.recv()
292 292 else:
293 293 line = self.process.stdout.readline()
294 294 # a stopped process will be readable but return empty strings
295 295 if line:
296 296 self.log.info(line[:-1])
297 297 else:
298 298 self.poll()
299 299
300 300 def handle_stderr(self, fd, events):
301 301 if WINDOWS:
302 302 line = self.stderr.recv()
303 303 else:
304 304 line = self.process.stderr.readline()
305 305 # a stopped process will be readable but return empty strings
306 306 if line:
307 307 self.log.error(line[:-1])
308 308 else:
309 309 self.poll()
310 310
311 311 def poll(self):
312 312 status = self.process.poll()
313 313 if status is not None:
314 314 self.poller.stop()
315 315 self.loop.remove_handler(self.stdout)
316 316 self.loop.remove_handler(self.stderr)
317 317 self.notify_stop(dict(exit_code=status, pid=self.process.pid))
318 318 return status
319 319
320 320 class LocalControllerLauncher(LocalProcessLauncher):
321 321 """Launch a controller as a regular external process."""
322 322
323 323 controller_cmd = List(ipcontroller_cmd_argv, config=True,
324 324 help="""Popen command to launch ipcontroller.""")
325 325 # Command line arguments to ipcontroller.
326 326 controller_args = List(['--log-to-file','--log-level=%i'%logging.INFO], config=True,
327 327 help="""command-line args to pass to ipcontroller""")
328 328
329 329 def find_args(self):
330 330 return self.controller_cmd + self.controller_args
331 331
332 332 def start(self, profile_dir):
333 333 """Start the controller by profile_dir."""
334 334 self.controller_args.extend(['--profile-dir=%s'%profile_dir])
335 335 self.profile_dir = unicode(profile_dir)
336 336 self.log.info("Starting LocalControllerLauncher: %r" % self.args)
337 337 return super(LocalControllerLauncher, self).start()
338 338
339 339
340 340 class LocalEngineLauncher(LocalProcessLauncher):
341 341 """Launch a single engine as a regular externall process."""
342 342
343 343 engine_cmd = List(ipengine_cmd_argv, config=True,
344 344 help="""command to launch the Engine.""")
345 345 # Command line arguments for ipengine.
346 346 engine_args = List(['--log-to-file','--log-level=%i'%logging.INFO], config=True,
347 347 help="command-line arguments to pass to ipengine"
348 348 )
349 349
350 350 def find_args(self):
351 351 return self.engine_cmd + self.engine_args
352 352
353 353 def start(self, profile_dir):
354 354 """Start the engine by profile_dir."""
355 355 self.engine_args.extend(['--profile-dir=%s'%profile_dir])
356 356 self.profile_dir = unicode(profile_dir)
357 357 return super(LocalEngineLauncher, self).start()
358 358
359 359
360 360 class LocalEngineSetLauncher(BaseLauncher):
361 361 """Launch a set of engines as regular external processes."""
362 362
363 363 # Command line arguments for ipengine.
364 364 engine_args = List(
365 365 ['--log-to-file','--log-level=%i'%logging.INFO], config=True,
366 366 help="command-line arguments to pass to ipengine"
367 367 )
368 368 # launcher class
369 369 launcher_class = LocalEngineLauncher
370 370
371 371 launchers = Dict()
372 372 stop_data = Dict()
373 373
374 374 def __init__(self, work_dir=u'.', config=None, **kwargs):
375 375 super(LocalEngineSetLauncher, self).__init__(
376 376 work_dir=work_dir, config=config, **kwargs
377 377 )
378 378 self.stop_data = {}
379 379
380 380 def start(self, n, profile_dir):
381 381 """Start n engines by profile or profile_dir."""
382 382 self.profile_dir = unicode(profile_dir)
383 383 dlist = []
384 384 for i in range(n):
385 385 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
386 386 # Copy the engine args over to each engine launcher.
387 387 el.engine_args = copy.deepcopy(self.engine_args)
388 388 el.on_stop(self._notice_engine_stopped)
389 389 d = el.start(profile_dir)
390 390 if i==0:
391 391 self.log.info("Starting LocalEngineSetLauncher: %r" % el.args)
392 392 self.launchers[i] = el
393 393 dlist.append(d)
394 394 self.notify_start(dlist)
395 395 # The consumeErrors here could be dangerous
396 396 # dfinal = gatherBoth(dlist, consumeErrors=True)
397 397 # dfinal.addCallback(self.notify_start)
398 398 return dlist
399 399
400 400 def find_args(self):
401 401 return ['engine set']
402 402
403 403 def signal(self, sig):
404 404 dlist = []
405 405 for el in self.launchers.itervalues():
406 406 d = el.signal(sig)
407 407 dlist.append(d)
408 408 # dfinal = gatherBoth(dlist, consumeErrors=True)
409 409 return dlist
410 410
411 411 def interrupt_then_kill(self, delay=1.0):
412 412 dlist = []
413 413 for el in self.launchers.itervalues():
414 414 d = el.interrupt_then_kill(delay)
415 415 dlist.append(d)
416 416 # dfinal = gatherBoth(dlist, consumeErrors=True)
417 417 return dlist
418 418
419 419 def stop(self):
420 420 return self.interrupt_then_kill()
421 421
422 422 def _notice_engine_stopped(self, data):
423 423 pid = data['pid']
424 424 for idx,el in self.launchers.iteritems():
425 425 if el.process.pid == pid:
426 426 break
427 427 self.launchers.pop(idx)
428 428 self.stop_data[idx] = data
429 429 if not self.launchers:
430 430 self.notify_stop(self.stop_data)
431 431
432 432
433 433 #-----------------------------------------------------------------------------
434 434 # MPIExec launchers
435 435 #-----------------------------------------------------------------------------
436 436
437 437
438 438 class MPIExecLauncher(LocalProcessLauncher):
439 439 """Launch an external process using mpiexec."""
440 440
441 441 mpi_cmd = List(['mpiexec'], config=True,
442 442 help="The mpiexec command to use in starting the process."
443 443 )
444 444 mpi_args = List([], config=True,
445 445 help="The command line arguments to pass to mpiexec."
446 446 )
447 447 program = List(['date'], config=True,
448 448 help="The program to start via mpiexec.")
449 449 program_args = List([], config=True,
450 450 help="The command line argument to the program."
451 451 )
452 452 n = Int(1)
453 453
454 454 def find_args(self):
455 455 """Build self.args using all the fields."""
456 456 return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
457 457 self.program + self.program_args
458 458
459 459 def start(self, n):
460 460 """Start n instances of the program using mpiexec."""
461 461 self.n = n
462 462 return super(MPIExecLauncher, self).start()
463 463
464 464
465 465 class MPIExecControllerLauncher(MPIExecLauncher):
466 466 """Launch a controller using mpiexec."""
467 467
468 468 controller_cmd = List(ipcontroller_cmd_argv, config=True,
469 469 help="Popen command to launch the Contropper"
470 470 )
471 471 controller_args = List(['--log-to-file','--log-level=%i'%logging.INFO], config=True,
472 472 help="Command line arguments to pass to ipcontroller."
473 473 )
474 474 n = Int(1)
475 475
476 476 def start(self, profile_dir):
477 477 """Start the controller by profile_dir."""
478 478 self.controller_args.extend(['--profile-dir=%s'%profile_dir])
479 479 self.profile_dir = unicode(profile_dir)
480 480 self.log.info("Starting MPIExecControllerLauncher: %r" % self.args)
481 481 return super(MPIExecControllerLauncher, self).start(1)
482 482
483 483 def find_args(self):
484 484 return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
485 485 self.controller_cmd + self.controller_args
486 486
487 487
488 488 class MPIExecEngineSetLauncher(MPIExecLauncher):
489 489
490 490 program = List(ipengine_cmd_argv, config=True,
491 491 help="Popen command for ipengine"
492 492 )
493 493 program_args = List(
494 494 ['--log-to-file','--log-level=%i'%logging.INFO], config=True,
495 495 help="Command line arguments for ipengine."
496 496 )
497 497 n = Int(1)
498 498
499 499 def start(self, n, profile_dir):
500 500 """Start n engines by profile or profile_dir."""
501 501 self.program_args.extend(['--profile-dir=%s'%profile_dir])
502 502 self.profile_dir = unicode(profile_dir)
503 503 self.n = n
504 504 self.log.info('Starting MPIExecEngineSetLauncher: %r' % self.args)
505 505 return super(MPIExecEngineSetLauncher, self).start(n)
506 506
507 507 #-----------------------------------------------------------------------------
508 508 # SSH launchers
509 509 #-----------------------------------------------------------------------------
510 510
511 511 # TODO: Get SSH Launcher back to level of sshx in 0.10.2
512 512
513 513 class SSHLauncher(LocalProcessLauncher):
514 514 """A minimal launcher for ssh.
515 515
516 516 To be useful this will probably have to be extended to use the ``sshx``
517 517 idea for environment variables. There could be other things this needs
518 518 as well.
519 519 """
520 520
521 521 ssh_cmd = List(['ssh'], config=True,
522 522 help="command for starting ssh")
523 523 ssh_args = List(['-tt'], config=True,
524 524 help="args to pass to ssh")
525 525 program = List(['date'], config=True,
526 526 help="Program to launch via ssh")
527 527 program_args = List([], config=True,
528 528 help="args to pass to remote program")
529 529 hostname = Unicode('', config=True,
530 530 help="hostname on which to launch the program")
531 531 user = Unicode('', config=True,
532 532 help="username for ssh")
533 533 location = Unicode('', config=True,
534 534 help="user@hostname location for ssh in one setting")
535 535
536 536 def _hostname_changed(self, name, old, new):
537 537 if self.user:
538 538 self.location = u'%s@%s' % (self.user, new)
539 539 else:
540 540 self.location = new
541 541
542 542 def _user_changed(self, name, old, new):
543 543 self.location = u'%s@%s' % (new, self.hostname)
544 544
545 545 def find_args(self):
546 546 return self.ssh_cmd + self.ssh_args + [self.location] + \
547 547 self.program + self.program_args
548 548
549 549 def start(self, profile_dir, hostname=None, user=None):
550 550 self.profile_dir = unicode(profile_dir)
551 551 if hostname is not None:
552 552 self.hostname = hostname
553 553 if user is not None:
554 554 self.user = user
555 555
556 556 return super(SSHLauncher, self).start()
557 557
558 558 def signal(self, sig):
559 559 if self.state == 'running':
560 560 # send escaped ssh connection-closer
561 561 self.process.stdin.write('~.')
562 562 self.process.stdin.flush()
563 563
564 564
565 565
566 566 class SSHControllerLauncher(SSHLauncher):
567 567
568 568 program = List(ipcontroller_cmd_argv, config=True,
569 569 help="remote ipcontroller command.")
570 570 program_args = List(['--reuse-files', '--log-to-file','--log-level=%i'%logging.INFO], config=True,
571 571 help="Command line arguments to ipcontroller.")
572 572
573 573
574 574 class SSHEngineLauncher(SSHLauncher):
575 575 program = List(ipengine_cmd_argv, config=True,
576 576 help="remote ipengine command.")
577 577 # Command line arguments for ipengine.
578 578 program_args = List(
579 ['--log-to-file','log_level=%i'%logging.INFO], config=True,
579 ['--log-to-file','--log_level=%i'%logging.INFO], config=True,
580 580 help="Command line arguments to ipengine."
581 581 )
582 582
583 583 class SSHEngineSetLauncher(LocalEngineSetLauncher):
584 584 launcher_class = SSHEngineLauncher
585 585 engines = Dict(config=True,
586 586 help="""dict of engines to launch. This is a dict by hostname of ints,
587 587 corresponding to the number of engines to start on that host.""")
588 588
589 589 def start(self, n, profile_dir):
590 590 """Start engines by profile or profile_dir.
591 591 `n` is ignored, and the `engines` config property is used instead.
592 592 """
593 593
594 594 self.profile_dir = unicode(profile_dir)
595 595 dlist = []
596 596 for host, n in self.engines.iteritems():
597 597 if isinstance(n, (tuple, list)):
598 598 n, args = n
599 599 else:
600 600 args = copy.deepcopy(self.engine_args)
601 601
602 602 if '@' in host:
603 603 user,host = host.split('@',1)
604 604 else:
605 605 user=None
606 606 for i in range(n):
607 607 el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log)
608 608
609 609 # Copy the engine args over to each engine launcher.
610 610 i
611 611 el.program_args = args
612 612 el.on_stop(self._notice_engine_stopped)
613 613 d = el.start(profile_dir, user=user, hostname=host)
614 614 if i==0:
615 615 self.log.info("Starting SSHEngineSetLauncher: %r" % el.args)
616 616 self.launchers[host+str(i)] = el
617 617 dlist.append(d)
618 618 self.notify_start(dlist)
619 619 return dlist
620 620
621 621
622 622
623 623 #-----------------------------------------------------------------------------
624 624 # Windows HPC Server 2008 scheduler launchers
625 625 #-----------------------------------------------------------------------------
626 626
627 627
628 628 # This is only used on Windows.
629 629 def find_job_cmd():
630 630 if WINDOWS:
631 631 try:
632 632 return find_cmd('job')
633 633 except (FindCmdError, ImportError):
634 634 # ImportError will be raised if win32api is not installed
635 635 return 'job'
636 636 else:
637 637 return 'job'
638 638
639 639
640 640 class WindowsHPCLauncher(BaseLauncher):
641 641
642 642 job_id_regexp = Unicode(r'\d+', config=True,
643 643 help="""A regular expression used to get the job id from the output of the
644 644 submit_command. """
645 645 )
646 646 job_file_name = Unicode(u'ipython_job.xml', config=True,
647 647 help="The filename of the instantiated job script.")
648 648 # The full path to the instantiated job script. This gets made dynamically
649 649 # by combining the work_dir with the job_file_name.
650 650 job_file = Unicode(u'')
651 651 scheduler = Unicode('', config=True,
652 652 help="The hostname of the scheduler to submit the job to.")
653 653 job_cmd = Unicode(find_job_cmd(), config=True,
654 654 help="The command for submitting jobs.")
655 655
656 656 def __init__(self, work_dir=u'.', config=None, **kwargs):
657 657 super(WindowsHPCLauncher, self).__init__(
658 658 work_dir=work_dir, config=config, **kwargs
659 659 )
660 660
661 661 @property
662 662 def job_file(self):
663 663 return os.path.join(self.work_dir, self.job_file_name)
664 664
665 665 def write_job_file(self, n):
666 666 raise NotImplementedError("Implement write_job_file in a subclass.")
667 667
668 668 def find_args(self):
669 669 return [u'job.exe']
670 670
671 671 def parse_job_id(self, output):
672 672 """Take the output of the submit command and return the job id."""
673 673 m = re.search(self.job_id_regexp, output)
674 674 if m is not None:
675 675 job_id = m.group()
676 676 else:
677 677 raise LauncherError("Job id couldn't be determined: %s" % output)
678 678 self.job_id = job_id
679 679 self.log.info('Job started with job id: %r' % job_id)
680 680 return job_id
681 681
682 682 def start(self, n):
683 683 """Start n copies of the process using the Win HPC job scheduler."""
684 684 self.write_job_file(n)
685 685 args = [
686 686 'submit',
687 687 '/jobfile:%s' % self.job_file,
688 688 '/scheduler:%s' % self.scheduler
689 689 ]
690 690 self.log.info("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
691 691
692 692 output = check_output([self.job_cmd]+args,
693 693 env=os.environ,
694 694 cwd=self.work_dir,
695 695 stderr=STDOUT
696 696 )
697 697 job_id = self.parse_job_id(output)
698 698 self.notify_start(job_id)
699 699 return job_id
700 700
701 701 def stop(self):
702 702 args = [
703 703 'cancel',
704 704 self.job_id,
705 705 '/scheduler:%s' % self.scheduler
706 706 ]
707 707 self.log.info("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
708 708 try:
709 709 output = check_output([self.job_cmd]+args,
710 710 env=os.environ,
711 711 cwd=self.work_dir,
712 712 stderr=STDOUT
713 713 )
714 714 except:
715 715 output = 'The job already appears to be stoppped: %r' % self.job_id
716 716 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
717 717 return output
718 718
719 719
720 720 class WindowsHPCControllerLauncher(WindowsHPCLauncher):
721 721
722 722 job_file_name = Unicode(u'ipcontroller_job.xml', config=True,
723 723 help="WinHPC xml job file.")
724 724 extra_args = List([], config=False,
725 725 help="extra args to pass to ipcontroller")
726 726
727 727 def write_job_file(self, n):
728 728 job = IPControllerJob(config=self.config)
729 729
730 730 t = IPControllerTask(config=self.config)
731 731 # The tasks work directory is *not* the actual work directory of
732 732 # the controller. It is used as the base path for the stdout/stderr
733 733 # files that the scheduler redirects to.
734 734 t.work_directory = self.profile_dir
735 735 # Add the profile_dir and from self.start().
736 736 t.controller_args.extend(self.extra_args)
737 737 job.add_task(t)
738 738
739 739 self.log.info("Writing job description file: %s" % self.job_file)
740 740 job.write(self.job_file)
741 741
742 742 @property
743 743 def job_file(self):
744 744 return os.path.join(self.profile_dir, self.job_file_name)
745 745
746 746 def start(self, profile_dir):
747 747 """Start the controller by profile_dir."""
748 748 self.extra_args = ['--profile-dir=%s'%profile_dir]
749 749 self.profile_dir = unicode(profile_dir)
750 750 return super(WindowsHPCControllerLauncher, self).start(1)
751 751
752 752
753 753 class WindowsHPCEngineSetLauncher(WindowsHPCLauncher):
754 754
755 755 job_file_name = Unicode(u'ipengineset_job.xml', config=True,
756 756 help="jobfile for ipengines job")
757 757 extra_args = List([], config=False,
758 758 help="extra args to pas to ipengine")
759 759
760 760 def write_job_file(self, n):
761 761 job = IPEngineSetJob(config=self.config)
762 762
763 763 for i in range(n):
764 764 t = IPEngineTask(config=self.config)
765 765 # The tasks work directory is *not* the actual work directory of
766 766 # the engine. It is used as the base path for the stdout/stderr
767 767 # files that the scheduler redirects to.
768 768 t.work_directory = self.profile_dir
769 769 # Add the profile_dir and from self.start().
770 770 t.engine_args.extend(self.extra_args)
771 771 job.add_task(t)
772 772
773 773 self.log.info("Writing job description file: %s" % self.job_file)
774 774 job.write(self.job_file)
775 775
776 776 @property
777 777 def job_file(self):
778 778 return os.path.join(self.profile_dir, self.job_file_name)
779 779
780 780 def start(self, n, profile_dir):
781 781 """Start the controller by profile_dir."""
782 782 self.extra_args = ['--profile-dir=%s'%profile_dir]
783 783 self.profile_dir = unicode(profile_dir)
784 784 return super(WindowsHPCEngineSetLauncher, self).start(n)
785 785
786 786
787 787 #-----------------------------------------------------------------------------
788 788 # Batch (PBS) system launchers
789 789 #-----------------------------------------------------------------------------
790 790
791 791 class BatchSystemLauncher(BaseLauncher):
792 792 """Launch an external process using a batch system.
793 793
794 794 This class is designed to work with UNIX batch systems like PBS, LSF,
795 795 GridEngine, etc. The overall model is that there are different commands
796 796 like qsub, qdel, etc. that handle the starting and stopping of the process.
797 797
798 798 This class also has the notion of a batch script. The ``batch_template``
799 799 attribute can be set to a string that is a template for the batch script.
800 800 This template is instantiated using string formatting. Thus the template can
801 801 use {n} fot the number of instances. Subclasses can add additional variables
802 802 to the template dict.
803 803 """
804 804
805 805 # Subclasses must fill these in. See PBSEngineSet
806 806 submit_command = List([''], config=True,
807 807 help="The name of the command line program used to submit jobs.")
808 808 delete_command = List([''], config=True,
809 809 help="The name of the command line program used to delete jobs.")
810 810 job_id_regexp = Unicode('', config=True,
811 811 help="""A regular expression used to get the job id from the output of the
812 812 submit_command.""")
813 813 batch_template = Unicode('', config=True,
814 814 help="The string that is the batch script template itself.")
815 815 batch_template_file = Unicode(u'', config=True,
816 816 help="The file that contains the batch template.")
817 817 batch_file_name = Unicode(u'batch_script', config=True,
818 818 help="The filename of the instantiated batch script.")
819 819 queue = Unicode(u'', config=True,
820 820 help="The PBS Queue.")
821 821
822 822 # not configurable, override in subclasses
823 823 # PBS Job Array regex
824 824 job_array_regexp = Unicode('')
825 825 job_array_template = Unicode('')
826 826 # PBS Queue regex
827 827 queue_regexp = Unicode('')
828 828 queue_template = Unicode('')
829 829 # The default batch template, override in subclasses
830 830 default_template = Unicode('')
831 831 # The full path to the instantiated batch script.
832 832 batch_file = Unicode(u'')
833 833 # the format dict used with batch_template:
834 834 context = Dict()
835 835 # the Formatter instance for rendering the templates:
836 836 formatter = Instance(EvalFormatter, (), {})
837 837
838 838
839 839 def find_args(self):
840 840 return self.submit_command + [self.batch_file]
841 841
842 842 def __init__(self, work_dir=u'.', config=None, **kwargs):
843 843 super(BatchSystemLauncher, self).__init__(
844 844 work_dir=work_dir, config=config, **kwargs
845 845 )
846 846 self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
847 847
848 848 def parse_job_id(self, output):
849 849 """Take the output of the submit command and return the job id."""
850 850 m = re.search(self.job_id_regexp, output)
851 851 if m is not None:
852 852 job_id = m.group()
853 853 else:
854 854 raise LauncherError("Job id couldn't be determined: %s" % output)
855 855 self.job_id = job_id
856 856 self.log.info('Job submitted with job id: %r' % job_id)
857 857 return job_id
858 858
859 859 def write_batch_script(self, n):
860 860 """Instantiate and write the batch script to the work_dir."""
861 861 self.context['n'] = n
862 862 self.context['queue'] = self.queue
863 863 # first priority is batch_template if set
864 864 if self.batch_template_file and not self.batch_template:
865 865 # second priority is batch_template_file
866 866 with open(self.batch_template_file) as f:
867 867 self.batch_template = f.read()
868 868 if not self.batch_template:
869 869 # third (last) priority is default_template
870 870 self.batch_template = self.default_template
871 871
872 872 # add jobarray or queue lines to user-specified template
873 873 # note that this is *only* when user did not specify a template.
874 874 regex = re.compile(self.job_array_regexp)
875 875 # print regex.search(self.batch_template)
876 876 if not regex.search(self.batch_template):
877 877 self.log.info("adding job array settings to batch script")
878 878 firstline, rest = self.batch_template.split('\n',1)
879 879 self.batch_template = u'\n'.join([firstline, self.job_array_template, rest])
880 880
881 881 regex = re.compile(self.queue_regexp)
882 882 # print regex.search(self.batch_template)
883 883 if self.queue and not regex.search(self.batch_template):
884 884 self.log.info("adding PBS queue settings to batch script")
885 885 firstline, rest = self.batch_template.split('\n',1)
886 886 self.batch_template = u'\n'.join([firstline, self.queue_template, rest])
887 887
888 888 script_as_string = self.formatter.format(self.batch_template, **self.context)
889 889 self.log.info('Writing instantiated batch script: %s' % self.batch_file)
890 890
891 891 with open(self.batch_file, 'w') as f:
892 892 f.write(script_as_string)
893 893 os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
894 894
895 895 def start(self, n, profile_dir):
896 896 """Start n copies of the process using a batch system."""
897 897 # Here we save profile_dir in the context so they
898 898 # can be used in the batch script template as {profile_dir}
899 899 self.context['profile_dir'] = profile_dir
900 900 self.profile_dir = unicode(profile_dir)
901 901 self.write_batch_script(n)
902 902 output = check_output(self.args, env=os.environ)
903 903
904 904 job_id = self.parse_job_id(output)
905 905 self.notify_start(job_id)
906 906 return job_id
907 907
908 908 def stop(self):
909 909 output = check_output(self.delete_command+[self.job_id], env=os.environ)
910 910 self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
911 911 return output
912 912
913 913
914 914 class PBSLauncher(BatchSystemLauncher):
915 915 """A BatchSystemLauncher subclass for PBS."""
916 916
917 917 submit_command = List(['qsub'], config=True,
918 918 help="The PBS submit command ['qsub']")
919 919 delete_command = List(['qdel'], config=True,
920 920 help="The PBS delete command ['qsub']")
921 921 job_id_regexp = Unicode(r'\d+', config=True,
922 922 help="Regular expresion for identifying the job ID [r'\d+']")
923 923
924 924 batch_file = Unicode(u'')
925 925 job_array_regexp = Unicode('#PBS\W+-t\W+[\w\d\-\$]+')
926 926 job_array_template = Unicode('#PBS -t 1-{n}')
927 927 queue_regexp = Unicode('#PBS\W+-q\W+\$?\w+')
928 928 queue_template = Unicode('#PBS -q {queue}')
929 929
930 930
931 931 class PBSControllerLauncher(PBSLauncher):
932 932 """Launch a controller using PBS."""
933 933
934 934 batch_file_name = Unicode(u'pbs_controller', config=True,
935 935 help="batch file name for the controller job.")
936 936 default_template= Unicode("""#!/bin/sh
937 937 #PBS -V
938 938 #PBS -N ipcontroller
939 939 %s --log-to-file --profile-dir={profile_dir}
940 940 """%(' '.join(ipcontroller_cmd_argv)))
941 941
942 942 def start(self, profile_dir):
943 943 """Start the controller by profile or profile_dir."""
944 944 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
945 945 return super(PBSControllerLauncher, self).start(1, profile_dir)
946 946
947 947
948 948 class PBSEngineSetLauncher(PBSLauncher):
949 949 """Launch Engines using PBS"""
950 950 batch_file_name = Unicode(u'pbs_engines', config=True,
951 951 help="batch file name for the engine(s) job.")
952 952 default_template= Unicode(u"""#!/bin/sh
953 953 #PBS -V
954 954 #PBS -N ipengine
955 955 %s --profile-dir={profile_dir}
956 956 """%(' '.join(ipengine_cmd_argv)))
957 957
958 958 def start(self, n, profile_dir):
959 959 """Start n engines by profile or profile_dir."""
960 960 self.log.info('Starting %i engines with PBSEngineSetLauncher: %r' % (n, self.args))
961 961 return super(PBSEngineSetLauncher, self).start(n, profile_dir)
962 962
963 963 #SGE is very similar to PBS
964 964
965 965 class SGELauncher(PBSLauncher):
966 966 """Sun GridEngine is a PBS clone with slightly different syntax"""
967 967 job_array_regexp = Unicode('#\$\W+\-t')
968 968 job_array_template = Unicode('#$ -t 1-{n}')
969 969 queue_regexp = Unicode('#\$\W+-q\W+\$?\w+')
970 970 queue_template = Unicode('#$ -q {queue}')
971 971
972 972 class SGEControllerLauncher(SGELauncher):
973 973 """Launch a controller using SGE."""
974 974
975 975 batch_file_name = Unicode(u'sge_controller', config=True,
976 976 help="batch file name for the ipontroller job.")
977 977 default_template= Unicode(u"""#$ -V
978 978 #$ -S /bin/sh
979 979 #$ -N ipcontroller
980 980 %s --log-to-file --profile-dir={profile_dir}
981 981 """%(' '.join(ipcontroller_cmd_argv)))
982 982
983 983 def start(self, profile_dir):
984 984 """Start the controller by profile or profile_dir."""
985 985 self.log.info("Starting PBSControllerLauncher: %r" % self.args)
986 986 return super(SGEControllerLauncher, self).start(1, profile_dir)
987 987
988 988 class SGEEngineSetLauncher(SGELauncher):
989 989 """Launch Engines with SGE"""
990 990 batch_file_name = Unicode(u'sge_engines', config=True,
991 991 help="batch file name for the engine(s) job.")
992 992 default_template = Unicode("""#$ -V
993 993 #$ -S /bin/sh
994 994 #$ -N ipengine
995 995 %s --profile-dir={profile_dir}
996 996 """%(' '.join(ipengine_cmd_argv)))
997 997
998 998 def start(self, n, profile_dir):
999 999 """Start n engines by profile or profile_dir."""
1000 1000 self.log.info('Starting %i engines with SGEEngineSetLauncher: %r' % (n, self.args))
1001 1001 return super(SGEEngineSetLauncher, self).start(n, profile_dir)
1002 1002
1003 1003
1004 1004 #-----------------------------------------------------------------------------
1005 1005 # A launcher for ipcluster itself!
1006 1006 #-----------------------------------------------------------------------------
1007 1007
1008 1008
1009 1009 class IPClusterLauncher(LocalProcessLauncher):
1010 1010 """Launch the ipcluster program in an external process."""
1011 1011
1012 1012 ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
1013 1013 help="Popen command for ipcluster")
1014 1014 ipcluster_args = List(
1015 1015 ['--clean-logs', '--log-to-file', '--log-level=%i'%logging.INFO], config=True,
1016 1016 help="Command line arguments to pass to ipcluster.")
1017 1017 ipcluster_subcommand = Unicode('start')
1018 1018 ipcluster_n = Int(2)
1019 1019
1020 1020 def find_args(self):
1021 1021 return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
1022 1022 ['--n=%i'%self.ipcluster_n] + self.ipcluster_args
1023 1023
1024 1024 def start(self):
1025 1025 self.log.info("Starting ipcluster: %r" % self.args)
1026 1026 return super(IPClusterLauncher, self).start()
1027 1027
1028 1028 #-----------------------------------------------------------------------------
1029 1029 # Collections of launchers
1030 1030 #-----------------------------------------------------------------------------
1031 1031
1032 1032 local_launchers = [
1033 1033 LocalControllerLauncher,
1034 1034 LocalEngineLauncher,
1035 1035 LocalEngineSetLauncher,
1036 1036 ]
1037 1037 mpi_launchers = [
1038 1038 MPIExecLauncher,
1039 1039 MPIExecControllerLauncher,
1040 1040 MPIExecEngineSetLauncher,
1041 1041 ]
1042 1042 ssh_launchers = [
1043 1043 SSHLauncher,
1044 1044 SSHControllerLauncher,
1045 1045 SSHEngineLauncher,
1046 1046 SSHEngineSetLauncher,
1047 1047 ]
1048 1048 winhpc_launchers = [
1049 1049 WindowsHPCLauncher,
1050 1050 WindowsHPCControllerLauncher,
1051 1051 WindowsHPCEngineSetLauncher,
1052 1052 ]
1053 1053 pbs_launchers = [
1054 1054 PBSLauncher,
1055 1055 PBSControllerLauncher,
1056 1056 PBSEngineSetLauncher,
1057 1057 ]
1058 1058 sge_launchers = [
1059 1059 SGELauncher,
1060 1060 SGEControllerLauncher,
1061 1061 SGEEngineSetLauncher,
1062 1062 ]
1063 1063 all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\
1064 1064 + pbs_launchers + sge_launchers
1065 1065
@@ -1,1385 +1,1385 b''
1 1 =================
2 2 IPython reference
3 3 =================
4 4
5 5 .. _command_line_options:
6 6
7 7 Command-line usage
8 8 ==================
9 9
10 10 You start IPython with the command::
11 11
12 12 $ ipython [options] files
13 13
14 14 If invoked with no options, it executes all the files listed in sequence
15 15 and drops you into the interpreter while still acknowledging any options
16 16 you may have set in your ipython_config.py. This behavior is different from
17 17 standard Python, which when called as python -i will only execute one
18 18 file and ignore your configuration setup.
19 19
20 20 Please note that some of the configuration options are not available at
21 21 the command line, simply because they are not practical here. Look into
22 22 your ipythonrc configuration file for details on those. This file is typically
23 23 installed in the IPYTHON_DIR directory. For Linux
24 24 users, this will be $HOME/.config/ipython, and for other users it will be
25 25 $HOME/.ipython. For Windows users, $HOME resolves to C:\\Documents and
26 26 Settings\\YourUserName in most instances.
27 27
28 28
29 29 Eventloop integration
30 30 ---------------------
31 31
32 32 Previously IPython had command line options for controlling GUI event loop
33 33 integration (-gthread, -qthread, -q4thread, -wthread, -pylab). As of IPython
34 34 version 0.11, these have been removed. Please see the new ``%gui``
35 35 magic command or :ref:`this section <gui_support>` for details on the new
36 36 interface, or specify the gui at the commandline::
37 37
38 38 $ ipython --gui=qt
39 39
40 40
41 41 Regular Options
42 42 ---------------
43 43
44 44 After the above threading options have been given, regular options can
45 45 follow in any order. All options can be abbreviated to their shortest
46 46 non-ambiguous form and are case-sensitive. One or two dashes can be
47 47 used. Some options have an alternate short form, indicated after a ``|``.
48 48
49 49 Most options can also be set from your ipythonrc configuration file. See
50 50 the provided example for more details on what the options do. Options
51 51 given at the command line override the values set in the ipythonrc file.
52 52
53 53 All options with a [no] prepended can be specified in negated form
54 54 (--no-option instead of --option) to turn the feature off.
55 55
56 56 ``-h, --help`` print a help message and exit.
57 57
58 58 ``--pylab, pylab=<name>``
59 59 See :ref:`Matplotlib support <matplotlib_support>`
60 60 for more details.
61 61
62 62 ``--autocall=<val>``
63 63 Make IPython automatically call any callable object even if you
64 64 didn't type explicit parentheses. For example, 'str 43' becomes
65 65 'str(43)' automatically. The value can be '0' to disable the feature,
66 66 '1' for smart autocall, where it is not applied if there are no more
67 67 arguments on the line, and '2' for full autocall, where all callable
68 68 objects are automatically called (even if no arguments are
69 69 present). The default is '1'.
70 70
71 71 ``--[no-]autoindent``
72 72 Turn automatic indentation on/off.
73 73
74 74 ``--[no-]automagic``
75 75 make magic commands automatic (without needing their first character
76 76 to be %). Type %magic at the IPython prompt for more information.
77 77
78 78 ``--[no-]autoedit_syntax``
79 79 When a syntax error occurs after editing a file, automatically
80 80 open the file to the trouble causing line for convenient
81 81 fixing.
82 82
83 83 ``--[no-]banner``
84 84 Print the initial information banner (default on).
85 85
86 86 ``--c=<command>``
87 87 execute the given command string. This is similar to the -c
88 88 option in the normal Python interpreter.
89 89
90 ``--cache_size=<n>``
90 ``--cache-size=<n>``
91 91 size of the output cache (maximum number of entries to hold in
92 92 memory). The default is 1000, you can change it permanently in your
93 93 config file. Setting it to 0 completely disables the caching system,
94 94 and the minimum value accepted is 20 (if you provide a value less than
95 95 20, it is reset to 0 and a warning is issued) This limit is defined
96 96 because otherwise you'll spend more time re-flushing a too small cache
97 97 than working.
98 98
99 99 ``--classic``
100 100 Gives IPython a similar feel to the classic Python
101 101 prompt.
102 102
103 103 ``--colors=<scheme>``
104 104 Color scheme for prompts and exception reporting. Currently
105 105 implemented: NoColor, Linux and LightBG.
106 106
107 107 ``--[no-]color_info``
108 108 IPython can display information about objects via a set of functions,
109 109 and optionally can use colors for this, syntax highlighting source
110 110 code and various other elements. However, because this information is
111 111 passed through a pager (like 'less') and many pagers get confused with
112 112 color codes, this option is off by default. You can test it and turn
113 113 it on permanently in your ipythonrc file if it works for you. As a
114 114 reference, the 'less' pager supplied with Mandrake 8.2 works ok, but
115 115 that in RedHat 7.2 doesn't.
116 116
117 117 Test it and turn it on permanently if it works with your
118 118 system. The magic function %color_info allows you to toggle this
119 119 interactively for testing.
120 120
121 121 ``--[no-]debug``
122 122 Show information about the loading process. Very useful to pin down
123 123 problems with your configuration files or to get details about
124 124 session restores.
125 125
126 126 ``--[no-]deep_reload``
127 127 IPython can use the deep_reload module which reloads changes in
128 128 modules recursively (it replaces the reload() function, so you don't
129 129 need to change anything to use it). deep_reload() forces a full
130 130 reload of modules whose code may have changed, which the default
131 131 reload() function does not.
132 132
133 133 When deep_reload is off, IPython will use the normal reload(),
134 134 but deep_reload will still be available as dreload(). This
135 135 feature is off by default [which means that you have both
136 136 normal reload() and dreload()].
137 137
138 138 ``--editor=<name>``
139 139 Which editor to use with the %edit command. By default,
140 140 IPython will honor your EDITOR environment variable (if not
141 141 set, vi is the Unix default and notepad the Windows one).
142 142 Since this editor is invoked on the fly by IPython and is
143 143 meant for editing small code snippets, you may want to use a
144 144 small, lightweight editor here (in case your default EDITOR is
145 145 something like Emacs).
146 146
147 147 ``--ipython_dir=<name>``
148 148 name of your IPython configuration directory IPYTHON_DIR. This
149 149 can also be specified through the environment variable
150 150 IPYTHON_DIR.
151 151
152 152 ``--logfile=<name>``
153 153 specify the name of your logfile.
154 154
155 155 This implies ``%logstart`` at the beginning of your session
156 156
157 157 generate a log file of all input. The file is named
158 158 ipython_log.py in your current directory (which prevents logs
159 159 from multiple IPython sessions from trampling each other). You
160 160 can use this to later restore a session by loading your
161 161 logfile with ``ipython --i ipython_log.py``
162 162
163 163 ``--logplay=<name>``
164 164
165 165 NOT AVAILABLE in 0.11
166 166
167 167 you can replay a previous log. For restoring a session as close as
168 168 possible to the state you left it in, use this option (don't just run
169 169 the logfile). With -logplay, IPython will try to reconstruct the
170 170 previous working environment in full, not just execute the commands in
171 171 the logfile.
172 172
173 173 When a session is restored, logging is automatically turned on
174 174 again with the name of the logfile it was invoked with (it is
175 175 read from the log header). So once you've turned logging on for
176 176 a session, you can quit IPython and reload it as many times as
177 177 you want and it will continue to log its history and restore
178 178 from the beginning every time.
179 179
180 180 Caveats: there are limitations in this option. The history
181 181 variables _i*,_* and _dh don't get restored properly. In the
182 182 future we will try to implement full session saving by writing
183 183 and retrieving a 'snapshot' of the memory state of IPython. But
184 184 our first attempts failed because of inherent limitations of
185 185 Python's Pickle module, so this may have to wait.
186 186
187 187 ``--[no-]messages``
188 188 Print messages which IPython collects about its startup
189 189 process (default on).
190 190
191 191 ``--[no-]pdb``
192 192 Automatically call the pdb debugger after every uncaught
193 193 exception. If you are used to debugging using pdb, this puts
194 194 you automatically inside of it after any call (either in
195 195 IPython or in code called by it) which triggers an exception
196 196 which goes uncaught.
197 197
198 198 ``--[no-]pprint``
199 199 ipython can optionally use the pprint (pretty printer) module
200 200 for displaying results. pprint tends to give a nicer display
201 201 of nested data structures. If you like it, you can turn it on
202 202 permanently in your config file (default off).
203 203
204 204 ``--profile=<name>``
205 205
206 206 Select the IPython profile by name.
207 207
208 208 This is a quick way to keep and load multiple
209 209 config files for different tasks, especially if you use the
210 210 include option of config files. You can keep a basic
211 211 :file:`IPYTHON_DIR/profile_default/ipython_config.py` file
212 212 and then have other 'profiles' which
213 213 include this one and load extra things for particular
214 214 tasks. For example:
215 215
216 216 1. $IPYTHON_DIR/profile_default : load basic things you always want.
217 217 2. $IPYTHON_DIR/profile_math : load (1) and basic math-related modules.
218 218 3. $IPYTHON_DIR/profile_numeric : load (1) and Numeric and plotting modules.
219 219
220 220 Since it is possible to create an endless loop by having
221 221 circular file inclusions, IPython will stop if it reaches 15
222 222 recursive inclusions.
223 223
224 224 ``InteractiveShell.prompt_in1=<string>``
225 225
226 226 Specify the string used for input prompts. Note that if you are using
227 227 numbered prompts, the number is represented with a '\#' in the
228 228 string. Don't forget to quote strings with spaces embedded in
229 229 them. Default: 'In [\#]:'. The :ref:`prompts section <prompts>`
230 230 discusses in detail all the available escapes to customize your
231 231 prompts.
232 232
233 233 ``InteractiveShell.prompt_in2=<string>``
234 234 Similar to the previous option, but used for the continuation
235 235 prompts. The special sequence '\D' is similar to '\#', but
236 236 with all digits replaced dots (so you can have your
237 237 continuation prompt aligned with your input prompt). Default:
238 238 ' .\D.:' (note three spaces at the start for alignment with
239 239 'In [\#]').
240 240
241 241 ``InteractiveShell.prompt_out=<string>``
242 242 String used for output prompts, also uses numbers like
243 243 prompt_in1. Default: 'Out[\#]:'
244 244
245 245 ``--quick``
246 246 start in bare bones mode (no config file loaded).
247 247
248 248 ``config_file=<name>``
249 249 name of your IPython resource configuration file. Normally
250 250 IPython loads ipython_config.py (from current directory) or
251 251 IPYTHON_DIR/profile_default.
252 252
253 253 If the loading of your config file fails, IPython starts with
254 254 a bare bones configuration (no modules loaded at all).
255 255
256 256 ``--[no-]readline``
257 257 use the readline library, which is needed to support name
258 258 completion and command history, among other things. It is
259 259 enabled by default, but may cause problems for users of
260 260 X/Emacs in Python comint or shell buffers.
261 261
262 262 Note that X/Emacs 'eterm' buffers (opened with M-x term) support
263 263 IPython's readline and syntax coloring fine, only 'emacs' (M-x
264 264 shell and C-c !) buffers do not.
265 265
266 266 ``--TerminalInteractiveShell.screen_length=<n>``
267 267 number of lines of your screen. This is used to control
268 268 printing of very long strings. Strings longer than this number
269 269 of lines will be sent through a pager instead of directly
270 270 printed.
271 271
272 272 The default value for this is 0, which means IPython will
273 273 auto-detect your screen size every time it needs to print certain
274 274 potentially long strings (this doesn't change the behavior of the
275 275 'print' keyword, it's only triggered internally). If for some
276 276 reason this isn't working well (it needs curses support), specify
277 277 it yourself. Otherwise don't change the default.
278 278
279 279 ``--TerminalInteractiveShell.separate_in=<string>``
280 280
281 281 separator before input prompts.
282 282 Default: '\n'
283 283
284 284 ``--TerminalInteractiveShell.separate_out=<string>``
285 285 separator before output prompts.
286 286 Default: nothing.
287 287
288 288 ``--TerminalInteractiveShell.separate_out2=<string>``
289 289 separator after output prompts.
290 290 Default: nothing.
291 291 For these three options, use the value 0 to specify no separator.
292 292
293 293 ``--nosep``
294 294 shorthand for setting the above separators to empty strings.
295 295
296 296 Simply removes all input/output separators.
297 297
298 298 ``--init``
299 299 allows you to initialize a profile dir for configuration when you
300 300 install a new version of IPython or want to use a new profile.
301 301 Since new versions may include new command line options or example
302 302 files, this copies updated config files. Note that you should probably
303 303 use %upgrade instead,it's a safer alternative.
304 304
305 305 ``--version`` print version information and exit.
306 306
307 307 ``--xmode=<modename>``
308 308
309 309 Mode for exception reporting.
310 310
311 311 Valid modes: Plain, Context and Verbose.
312 312
313 313 * Plain: similar to python's normal traceback printing.
314 314 * Context: prints 5 lines of context source code around each
315 315 line in the traceback.
316 316 * Verbose: similar to Context, but additionally prints the
317 317 variables currently visible where the exception happened
318 318 (shortening their strings if too long). This can potentially be
319 319 very slow, if you happen to have a huge data structure whose
320 320 string representation is complex to compute. Your computer may
321 321 appear to freeze for a while with cpu usage at 100%. If this
322 322 occurs, you can cancel the traceback with Ctrl-C (maybe hitting it
323 323 more than once).
324 324
325 325 Interactive use
326 326 ===============
327 327
328 328 IPython is meant to work as a drop-in
329 329 replacement for the standard interactive interpreter. As such, any code
330 330 which is valid python should execute normally under IPython (cases where
331 331 this is not true should be reported as bugs). It does, however, offer
332 332 many features which are not available at a standard python prompt. What
333 333 follows is a list of these.
334 334
335 335
336 336 Caution for Windows users
337 337 -------------------------
338 338
339 339 Windows, unfortunately, uses the '\\' character as a path
340 340 separator. This is a terrible choice, because '\\' also represents the
341 341 escape character in most modern programming languages, including
342 342 Python. For this reason, using '/' character is recommended if you
343 343 have problems with ``\``. However, in Windows commands '/' flags
344 344 options, so you can not use it for the root directory. This means that
345 345 paths beginning at the root must be typed in a contrived manner like:
346 346 ``%copy \opt/foo/bar.txt \tmp``
347 347
348 348 .. _magic:
349 349
350 350 Magic command system
351 351 --------------------
352 352
353 353 IPython will treat any line whose first character is a % as a special
354 354 call to a 'magic' function. These allow you to control the behavior of
355 355 IPython itself, plus a lot of system-type features. They are all
356 356 prefixed with a % character, but parameters are given without
357 357 parentheses or quotes.
358 358
359 359 Example: typing ``%cd mydir`` changes your working directory to 'mydir', if it
360 360 exists.
361 361
362 362 If you have 'automagic' enabled (as it by default), you don't need
363 363 to type in the % explicitly. IPython will scan its internal list of
364 364 magic functions and call one if it exists. With automagic on you can
365 365 then just type ``cd mydir`` to go to directory 'mydir'. The automagic
366 366 system has the lowest possible precedence in name searches, so defining
367 367 an identifier with the same name as an existing magic function will
368 368 shadow it for automagic use. You can still access the shadowed magic
369 369 function by explicitly using the % character at the beginning of the line.
370 370
371 371 An example (with automagic on) should clarify all this:
372 372
373 373 .. sourcecode:: ipython
374 374
375 375 In [1]: cd ipython # %cd is called by automagic
376 376
377 377 /home/fperez/ipython
378 378
379 379 In [2]: cd=1 # now cd is just a variable
380 380
381 381 In [3]: cd .. # and doesn't work as a function anymore
382 382
383 383 ------------------------------
384 384
385 385 File "<console>", line 1
386 386
387 387 cd ..
388 388
389 389 ^
390 390
391 391 SyntaxError: invalid syntax
392 392
393 393 In [4]: %cd .. # but %cd always works
394 394
395 395 /home/fperez
396 396
397 397 In [5]: del cd # if you remove the cd variable
398 398
399 399 In [6]: cd ipython # automagic can work again
400 400
401 401 /home/fperez/ipython
402 402
403 403 You can define your own magic functions to extend the system. The
404 404 following example defines a new magic command, %impall:
405 405
406 406 .. sourcecode:: python
407 407
408 408 ip = get_ipython()
409 409
410 410 def doimp(self, arg):
411 411
412 412 ip = self.api
413 413
414 414 ip.ex("import %s; reload(%s); from %s import *" % (
415 415
416 416 arg,arg,arg)
417 417
418 418 )
419 419
420 420 ip.expose_magic('impall', doimp)
421 421
422 422 Type %magic for more information, including a list of all available
423 423 magic functions at any time and their docstrings. You can also type
424 424 %magic_function_name? (see sec. 6.4 <#sec:dyn-object-info> for
425 425 information on the '?' system) to get information about any particular
426 426 magic function you are interested in.
427 427
428 428 The API documentation for the :mod:`IPython.core.magic` module contains the full
429 429 docstrings of all currently available magic commands.
430 430
431 431
432 432 Access to the standard Python help
433 433 ----------------------------------
434 434
435 435 As of Python 2.1, a help system is available with access to object docstrings
436 436 and the Python manuals. Simply type 'help' (no quotes) to access it. You can
437 437 also type help(object) to obtain information about a given object, and
438 438 help('keyword') for information on a keyword. As noted :ref:`here
439 439 <accessing_help>`, you need to properly configure your environment variable
440 440 PYTHONDOCS for this feature to work correctly.
441 441
442 442 .. _dynamic_object_info:
443 443
444 444 Dynamic object information
445 445 --------------------------
446 446
447 447 Typing ?word or word? prints detailed information about an object. If
448 448 certain strings in the object are too long (docstrings, code, etc.) they
449 449 get snipped in the center for brevity. This system gives access variable
450 450 types and values, full source code for any object (if available),
451 451 function prototypes and other useful information.
452 452
453 453 Typing ??word or word?? gives access to the full information without
454 454 snipping long strings. Long strings are sent to the screen through the
455 455 less pager if longer than the screen and printed otherwise. On systems
456 456 lacking the less command, IPython uses a very basic internal pager.
457 457
458 458 The following magic functions are particularly useful for gathering
459 459 information about your working environment. You can get more details by
460 460 typing %magic or querying them individually (use %function_name? with or
461 461 without the %), this is just a summary:
462 462
463 463 * **%pdoc <object>**: Print (or run through a pager if too long) the
464 464 docstring for an object. If the given object is a class, it will
465 465 print both the class and the constructor docstrings.
466 466 * **%pdef <object>**: Print the definition header for any callable
467 467 object. If the object is a class, print the constructor information.
468 468 * **%psource <object>**: Print (or run through a pager if too long)
469 469 the source code for an object.
470 470 * **%pfile <object>**: Show the entire source file where an object was
471 471 defined via a pager, opening it at the line where the object
472 472 definition begins.
473 473 * **%who/%whos**: These functions give information about identifiers
474 474 you have defined interactively (not things you loaded or defined
475 475 in your configuration files). %who just prints a list of
476 476 identifiers and %whos prints a table with some basic details about
477 477 each identifier.
478 478
479 479 Note that the dynamic object information functions (?/??, %pdoc, %pfile,
480 480 %pdef, %psource) give you access to documentation even on things which
481 481 are not really defined as separate identifiers. Try for example typing
482 482 {}.get? or after doing import os, type os.path.abspath??.
483 483
484 484
485 485 .. _readline:
486 486
487 487 Readline-based features
488 488 -----------------------
489 489
490 490 These features require the GNU readline library, so they won't work if
491 491 your Python installation lacks readline support. We will first describe
492 492 the default behavior IPython uses, and then how to change it to suit
493 493 your preferences.
494 494
495 495
496 496 Command line completion
497 497 +++++++++++++++++++++++
498 498
499 499 At any time, hitting TAB will complete any available python commands or
500 500 variable names, and show you a list of the possible completions if
501 501 there's no unambiguous one. It will also complete filenames in the
502 502 current directory if no python names match what you've typed so far.
503 503
504 504
505 505 Search command history
506 506 ++++++++++++++++++++++
507 507
508 508 IPython provides two ways for searching through previous input and thus
509 509 reduce the need for repetitive typing:
510 510
511 511 1. Start typing, and then use Ctrl-p (previous,up) and Ctrl-n
512 512 (next,down) to search through only the history items that match
513 513 what you've typed so far. If you use Ctrl-p/Ctrl-n at a blank
514 514 prompt, they just behave like normal arrow keys.
515 515 2. Hit Ctrl-r: opens a search prompt. Begin typing and the system
516 516 searches your history for lines that contain what you've typed so
517 517 far, completing as much as it can.
518 518
519 519
520 520 Persistent command history across sessions
521 521 ++++++++++++++++++++++++++++++++++++++++++
522 522
523 523 IPython will save your input history when it leaves and reload it next
524 524 time you restart it. By default, the history file is named
525 525 $IPYTHON_DIR/profile_<name>/history.sqlite. This allows you to keep
526 526 separate histories related to various tasks: commands related to
527 527 numerical work will not be clobbered by a system shell history, for
528 528 example.
529 529
530 530
531 531 Autoindent
532 532 ++++++++++
533 533
534 534 IPython can recognize lines ending in ':' and indent the next line,
535 535 while also un-indenting automatically after 'raise' or 'return'.
536 536
537 537 This feature uses the readline library, so it will honor your ~/.inputrc
538 538 configuration (or whatever file your INPUTRC variable points to). Adding
539 539 the following lines to your .inputrc file can make indenting/unindenting
540 540 more convenient (M-i indents, M-u unindents)::
541 541
542 542 $if Python
543 543 "\M-i": " "
544 544 "\M-u": "\d\d\d\d"
545 545 $endif
546 546
547 547 Note that there are 4 spaces between the quote marks after "M-i" above.
548 548
549 549 .. warning::
550 550
551 551 Setting the above indents will cause problems with unicode text entry in the terminal.
552 552
553 553 .. warning::
554 554
555 555 Autoindent is ON by default, but it can cause problems with
556 556 the pasting of multi-line indented code (the pasted code gets
557 557 re-indented on each line). A magic function %autoindent allows you to
558 558 toggle it on/off at runtime. You can also disable it permanently on in
559 559 your :file:`ipython_config.py` file (set TerminalInteractiveShell.autoindent=False).
560 560
561 561 If you want to paste multiple lines, it is recommended that you use ``%paste``.
562 562
563 563
564 564 Customizing readline behavior
565 565 +++++++++++++++++++++++++++++
566 566
567 567 All these features are based on the GNU readline library, which has an
568 568 extremely customizable interface. Normally, readline is configured via a
569 569 file which defines the behavior of the library; the details of the
570 570 syntax for this can be found in the readline documentation available
571 571 with your system or on the Internet. IPython doesn't read this file (if
572 572 it exists) directly, but it does support passing to readline valid
573 573 options via a simple interface. In brief, you can customize readline by
574 574 setting the following options in your ipythonrc configuration file (note
575 575 that these options can not be specified at the command line):
576 576
577 577 * **readline_parse_and_bind**: this option can appear as many times as
578 578 you want, each time defining a string to be executed via a
579 579 readline.parse_and_bind() command. The syntax for valid commands
580 580 of this kind can be found by reading the documentation for the GNU
581 581 readline library, as these commands are of the kind which readline
582 582 accepts in its configuration file.
583 583 * **readline_remove_delims**: a string of characters to be removed
584 584 from the default word-delimiters list used by readline, so that
585 585 completions may be performed on strings which contain them. Do not
586 586 change the default value unless you know what you're doing.
587 587 * **readline_omit__names**: when tab-completion is enabled, hitting
588 588 <tab> after a '.' in a name will complete all attributes of an
589 589 object, including all the special methods whose names include
590 590 double underscores (like __getitem__ or __class__). If you'd
591 591 rather not see these names by default, you can set this option to
592 592 1. Note that even when this option is set, you can still see those
593 593 names by explicitly typing a _ after the period and hitting <tab>:
594 594 'name._<tab>' will always complete attribute names starting with '_'.
595 595
596 596 This option is off by default so that new users see all
597 597 attributes of any objects they are dealing with.
598 598
599 599 You will find the default values along with a corresponding detailed
600 600 explanation in your ipythonrc file.
601 601
602 602
603 603 Session logging and restoring
604 604 -----------------------------
605 605
606 606 You can log all input from a session either by starting IPython with the
607 607 command line switche ``logfile=foo.py`` (see :ref:`here <command_line_options>`)
608 608 or by activating the logging at any moment with the magic function %logstart.
609 609
610 610 Log files can later be reloaded by running them as scripts and IPython
611 611 will attempt to 'replay' the log by executing all the lines in it, thus
612 612 restoring the state of a previous session. This feature is not quite
613 613 perfect, but can still be useful in many cases.
614 614
615 615 The log files can also be used as a way to have a permanent record of
616 616 any code you wrote while experimenting. Log files are regular text files
617 617 which you can later open in your favorite text editor to extract code or
618 618 to 'clean them up' before using them to replay a session.
619 619
620 620 The %logstart function for activating logging in mid-session is used as
621 621 follows:
622 622
623 623 %logstart [log_name [log_mode]]
624 624
625 625 If no name is given, it defaults to a file named 'ipython_log.py' in your
626 626 current working directory, in 'rotate' mode (see below).
627 627
628 628 '%logstart name' saves to file 'name' in 'backup' mode. It saves your
629 629 history up to that point and then continues logging.
630 630
631 631 %logstart takes a second optional parameter: logging mode. This can be
632 632 one of (note that the modes are given unquoted):
633 633
634 634 * [over:] overwrite existing log_name.
635 635 * [backup:] rename (if exists) to log_name~ and start log_name.
636 636 * [append:] well, that says it.
637 637 * [rotate:] create rotating logs log_name.1~, log_name.2~, etc.
638 638
639 639 The %logoff and %logon functions allow you to temporarily stop and
640 640 resume logging to a file which had previously been started with
641 641 %logstart. They will fail (with an explanation) if you try to use them
642 642 before logging has been started.
643 643
644 644 .. _system_shell_access:
645 645
646 646 System shell access
647 647 -------------------
648 648
649 649 Any input line beginning with a ! character is passed verbatim (minus
650 650 the !, of course) to the underlying operating system. For example,
651 651 typing !ls will run 'ls' in the current directory.
652 652
653 653 Manual capture of command output
654 654 --------------------------------
655 655
656 656 If the input line begins with two exclamation marks, !!, the command is
657 657 executed but its output is captured and returned as a python list, split
658 658 on newlines. Any output sent by the subprocess to standard error is
659 659 printed separately, so that the resulting list only captures standard
660 660 output. The !! syntax is a shorthand for the %sx magic command.
661 661
662 662 Finally, the %sc magic (short for 'shell capture') is similar to %sx,
663 663 but allowing more fine-grained control of the capture details, and
664 664 storing the result directly into a named variable. The direct use of
665 665 %sc is now deprecated, and you should ise the ``var = !cmd`` syntax
666 666 instead.
667 667
668 668 IPython also allows you to expand the value of python variables when
669 669 making system calls. Any python variable or expression which you prepend
670 670 with $ will get expanded before the system call is made::
671 671
672 672 In [1]: pyvar='Hello world'
673 673 In [2]: !echo "A python variable: $pyvar"
674 674 A python variable: Hello world
675 675
676 676 If you want the shell to actually see a literal $, you need to type it
677 677 twice::
678 678
679 679 In [3]: !echo "A system variable: $$HOME"
680 680 A system variable: /home/fperez
681 681
682 682 You can pass arbitrary expressions, though you'll need to delimit them
683 683 with {} if there is ambiguity as to the extent of the expression::
684 684
685 685 In [5]: x=10
686 686 In [6]: y=20
687 687 In [13]: !echo $x+y
688 688 10+y
689 689 In [7]: !echo ${x+y}
690 690 30
691 691
692 692 Even object attributes can be expanded::
693 693
694 694 In [12]: !echo $sys.argv
695 695 [/home/fperez/usr/bin/ipython]
696 696
697 697
698 698 System command aliases
699 699 ----------------------
700 700
701 701 The %alias magic function and the alias option in the ipythonrc
702 702 configuration file allow you to define magic functions which are in fact
703 703 system shell commands. These aliases can have parameters.
704 704
705 705 '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
706 706
707 707 Then, typing '%alias_name params' will execute the system command 'cmd
708 708 params' (from your underlying operating system).
709 709
710 710 You can also define aliases with parameters using %s specifiers (one per
711 711 parameter). The following example defines the %parts function as an
712 712 alias to the command 'echo first %s second %s' where each %s will be
713 713 replaced by a positional parameter to the call to %parts::
714 714
715 715 In [1]: alias parts echo first %s second %s
716 716 In [2]: %parts A B
717 717 first A second B
718 718 In [3]: %parts A
719 719 Incorrect number of arguments: 2 expected.
720 720 parts is an alias to: 'echo first %s second %s'
721 721
722 722 If called with no parameters, %alias prints the table of currently
723 723 defined aliases.
724 724
725 725 The %rehash/rehashx magics allow you to load your entire $PATH as
726 726 ipython aliases. See their respective docstrings (or sec. 6.2
727 727 <#sec:magic> for further details).
728 728
729 729
730 730 .. _dreload:
731 731
732 732 Recursive reload
733 733 ----------------
734 734
735 735 The dreload function does a recursive reload of a module: changes made
736 736 to the module since you imported will actually be available without
737 737 having to exit.
738 738
739 739
740 740 Verbose and colored exception traceback printouts
741 741 -------------------------------------------------
742 742
743 743 IPython provides the option to see very detailed exception tracebacks,
744 744 which can be especially useful when debugging large programs. You can
745 745 run any Python file with the %run function to benefit from these
746 746 detailed tracebacks. Furthermore, both normal and verbose tracebacks can
747 747 be colored (if your terminal supports it) which makes them much easier
748 748 to parse visually.
749 749
750 750 See the magic xmode and colors functions for details (just type %magic).
751 751
752 752 These features are basically a terminal version of Ka-Ping Yee's cgitb
753 753 module, now part of the standard Python library.
754 754
755 755
756 756 .. _input_caching:
757 757
758 758 Input caching system
759 759 --------------------
760 760
761 761 IPython offers numbered prompts (In/Out) with input and output caching
762 762 (also referred to as 'input history'). All input is saved and can be
763 763 retrieved as variables (besides the usual arrow key recall), in
764 764 addition to the %rep magic command that brings a history entry
765 765 up for editing on the next command line.
766 766
767 767 The following GLOBAL variables always exist (so don't overwrite them!):
768 768 _i: stores previous input. _ii: next previous. _iii: next-next previous.
769 769 _ih : a list of all input _ih[n] is the input from line n and this list
770 770 is aliased to the global variable In. If you overwrite In with a
771 771 variable of your own, you can remake the assignment to the internal list
772 772 with a simple 'In=_ih'.
773 773
774 774 Additionally, global variables named _i<n> are dynamically created (<n>
775 775 being the prompt counter), such that
776 776 _i<n> == _ih[<n>] == In[<n>].
777 777
778 778 For example, what you typed at prompt 14 is available as _i14, _ih[14]
779 779 and In[14].
780 780
781 781 This allows you to easily cut and paste multi line interactive prompts
782 782 by printing them out: they print like a clean string, without prompt
783 783 characters. You can also manipulate them like regular variables (they
784 784 are strings), modify or exec them (typing 'exec _i9' will re-execute the
785 785 contents of input prompt 9, 'exec In[9:14]+In[18]' will re-execute lines
786 786 9 through 13 and line 18).
787 787
788 788 You can also re-execute multiple lines of input easily by using the
789 789 magic %macro function (which automates the process and allows
790 790 re-execution without having to type 'exec' every time). The macro system
791 791 also allows you to re-execute previous lines which include magic
792 792 function calls (which require special processing). Type %macro? or see
793 793 sec. 6.2 <#sec:magic> for more details on the macro system.
794 794
795 795 A history function %hist allows you to see any part of your input
796 796 history by printing a range of the _i variables.
797 797
798 798 You can also search ('grep') through your history by typing
799 799 '%hist -g somestring'. This also searches through the so called *shadow history*,
800 800 which remembers all the commands (apart from multiline code blocks)
801 801 you have ever entered. Handy for searching for svn/bzr URL's, IP adrresses
802 802 etc. You can bring shadow history entries listed by '%hist -g' up for editing
803 803 (or re-execution by just pressing ENTER) with %rep command. Shadow history
804 804 entries are not available as _iNUMBER variables, and they are identified by
805 805 the '0' prefix in %hist -g output. That is, history entry 12 is a normal
806 806 history entry, but 0231 is a shadow history entry.
807 807
808 808 Shadow history was added because the readline history is inherently very
809 809 unsafe - if you have multiple IPython sessions open, the last session
810 810 to close will overwrite the history of previountly closed session. Likewise,
811 811 if a crash occurs, history is never saved, whereas shadow history entries
812 812 are added after entering every command (so a command executed
813 813 in another IPython session is immediately available in other IPython
814 814 sessions that are open).
815 815
816 816 To conserve space, a command can exist in shadow history only once - it doesn't
817 817 make sense to store a common line like "cd .." a thousand times. The idea is
818 818 mainly to provide a reliable place where valuable, hard-to-remember commands can
819 819 always be retrieved, as opposed to providing an exact sequence of commands
820 820 you have entered in actual order.
821 821
822 822 Because shadow history has all the commands you have ever executed,
823 823 time taken by %hist -g will increase oven time. If it ever starts to take
824 824 too long (or it ends up containing sensitive information like passwords),
825 825 clear the shadow history by `%clear shadow_nuke`.
826 826
827 827 Time taken to add entries to shadow history should be negligible, but
828 828 in any case, if you start noticing performance degradation after using
829 829 IPython for a long time (or running a script that floods the shadow history!),
830 830 you can 'compress' the shadow history by executing
831 831 `%clear shadow_compress`. In practice, this should never be necessary
832 832 in normal use.
833 833
834 834 .. _output_caching:
835 835
836 836 Output caching system
837 837 ---------------------
838 838
839 839 For output that is returned from actions, a system similar to the input
840 840 cache exists but using _ instead of _i. Only actions that produce a
841 841 result (NOT assignments, for example) are cached. If you are familiar
842 842 with Mathematica, IPython's _ variables behave exactly like
843 843 Mathematica's % variables.
844 844
845 845 The following GLOBAL variables always exist (so don't overwrite them!):
846 846
847 847 * [_] (a single underscore) : stores previous output, like Python's
848 848 default interpreter.
849 849 * [__] (two underscores): next previous.
850 850 * [___] (three underscores): next-next previous.
851 851
852 852 Additionally, global variables named _<n> are dynamically created (<n>
853 853 being the prompt counter), such that the result of output <n> is always
854 854 available as _<n> (don't use the angle brackets, just the number, e.g.
855 855 _21).
856 856
857 857 These global variables are all stored in a global dictionary (not a
858 858 list, since it only has entries for lines which returned a result)
859 859 available under the names _oh and Out (similar to _ih and In). So the
860 860 output from line 12 can be obtained as _12, Out[12] or _oh[12]. If you
861 861 accidentally overwrite the Out variable you can recover it by typing
862 862 'Out=_oh' at the prompt.
863 863
864 864 This system obviously can potentially put heavy memory demands on your
865 865 system, since it prevents Python's garbage collector from removing any
866 866 previously computed results. You can control how many results are kept
867 867 in memory with the option (at the command line or in your ipythonrc
868 868 file) cache_size. If you set it to 0, the whole system is completely
869 869 disabled and the prompts revert to the classic '>>>' of normal Python.
870 870
871 871
872 872 Directory history
873 873 -----------------
874 874
875 875 Your history of visited directories is kept in the global list _dh, and
876 876 the magic %cd command can be used to go to any entry in that list. The
877 877 %dhist command allows you to view this history. Do ``cd -<TAB`` to
878 878 conveniently view the directory history.
879 879
880 880
881 881 Automatic parentheses and quotes
882 882 --------------------------------
883 883
884 884 These features were adapted from Nathan Gray's LazyPython. They are
885 885 meant to allow less typing for common situations.
886 886
887 887
888 888 Automatic parentheses
889 889 ---------------------
890 890
891 891 Callable objects (i.e. functions, methods, etc) can be invoked like this
892 892 (notice the commas between the arguments)::
893 893
894 894 >>> callable_ob arg1, arg2, arg3
895 895
896 896 and the input will be translated to this::
897 897
898 898 -> callable_ob(arg1, arg2, arg3)
899 899
900 900 You can force automatic parentheses by using '/' as the first character
901 901 of a line. For example::
902 902
903 903 >>> /globals # becomes 'globals()'
904 904
905 905 Note that the '/' MUST be the first character on the line! This won't work::
906 906
907 907 >>> print /globals # syntax error
908 908
909 909 In most cases the automatic algorithm should work, so you should rarely
910 910 need to explicitly invoke /. One notable exception is if you are trying
911 911 to call a function with a list of tuples as arguments (the parenthesis
912 912 will confuse IPython)::
913 913
914 914 In [1]: zip (1,2,3),(4,5,6) # won't work
915 915
916 916 but this will work::
917 917
918 918 In [2]: /zip (1,2,3),(4,5,6)
919 919 ---> zip ((1,2,3),(4,5,6))
920 920 Out[2]= [(1, 4), (2, 5), (3, 6)]
921 921
922 922 IPython tells you that it has altered your command line by displaying
923 923 the new command line preceded by ->. e.g.::
924 924
925 925 In [18]: callable list
926 926 ----> callable (list)
927 927
928 928
929 929 Automatic quoting
930 930 -----------------
931 931
932 932 You can force automatic quoting of a function's arguments by using ','
933 933 or ';' as the first character of a line. For example::
934 934
935 935 >>> ,my_function /home/me # becomes my_function("/home/me")
936 936
937 937 If you use ';' instead, the whole argument is quoted as a single string
938 938 (while ',' splits on whitespace)::
939 939
940 940 >>> ,my_function a b c # becomes my_function("a","b","c")
941 941
942 942 >>> ;my_function a b c # becomes my_function("a b c")
943 943
944 944 Note that the ',' or ';' MUST be the first character on the line! This
945 945 won't work::
946 946
947 947 >>> x = ,my_function /home/me # syntax error
948 948
949 949 IPython as your default Python environment
950 950 ==========================================
951 951
952 952 Python honors the environment variable PYTHONSTARTUP and will execute at
953 953 startup the file referenced by this variable. If you put at the end of
954 954 this file the following two lines of code::
955 955
956 956 from IPython.frontend.terminal.ipapp import launch_new_instance
957 957 launch_new_instance()
958 958 raise SystemExit
959 959
960 960 then IPython will be your working environment anytime you start Python.
961 961 The ``raise SystemExit`` is needed to exit Python when
962 962 it finishes, otherwise you'll be back at the normal Python '>>>'
963 963 prompt.
964 964
965 965 This is probably useful to developers who manage multiple Python
966 966 versions and don't want to have correspondingly multiple IPython
967 967 versions. Note that in this mode, there is no way to pass IPython any
968 968 command-line options, as those are trapped first by Python itself.
969 969
970 970 .. _Embedding:
971 971
972 972 Embedding IPython
973 973 =================
974 974
975 975 It is possible to start an IPython instance inside your own Python
976 976 programs. This allows you to evaluate dynamically the state of your
977 977 code, operate with your variables, analyze them, etc. Note however that
978 978 any changes you make to values while in the shell do not propagate back
979 979 to the running code, so it is safe to modify your values because you
980 980 won't break your code in bizarre ways by doing so.
981 981
982 982 This feature allows you to easily have a fully functional python
983 983 environment for doing object introspection anywhere in your code with a
984 984 simple function call. In some cases a simple print statement is enough,
985 985 but if you need to do more detailed analysis of a code fragment this
986 986 feature can be very valuable.
987 987
988 988 It can also be useful in scientific computing situations where it is
989 989 common to need to do some automatic, computationally intensive part and
990 990 then stop to look at data, plots, etc.
991 991 Opening an IPython instance will give you full access to your data and
992 992 functions, and you can resume program execution once you are done with
993 993 the interactive part (perhaps to stop again later, as many times as
994 994 needed).
995 995
996 996 The following code snippet is the bare minimum you need to include in
997 997 your Python programs for this to work (detailed examples follow later)::
998 998
999 999 from IPython import embed
1000 1000
1001 1001 embed() # this call anywhere in your program will start IPython
1002 1002
1003 1003 You can run embedded instances even in code which is itself being run at
1004 1004 the IPython interactive prompt with '%run <filename>'. Since it's easy
1005 1005 to get lost as to where you are (in your top-level IPython or in your
1006 1006 embedded one), it's a good idea in such cases to set the in/out prompts
1007 1007 to something different for the embedded instances. The code examples
1008 1008 below illustrate this.
1009 1009
1010 1010 You can also have multiple IPython instances in your program and open
1011 1011 them separately, for example with different options for data
1012 1012 presentation. If you close and open the same instance multiple times,
1013 1013 its prompt counters simply continue from each execution to the next.
1014 1014
1015 1015 Please look at the docstrings in the :mod:`~IPython.frontend.terminal.embed`
1016 1016 module for more details on the use of this system.
1017 1017
1018 1018 The following sample file illustrating how to use the embedding
1019 1019 functionality is provided in the examples directory as example-embed.py.
1020 1020 It should be fairly self-explanatory:
1021 1021
1022 1022 .. literalinclude:: ../../examples/core/example-embed.py
1023 1023 :language: python
1024 1024
1025 1025 Once you understand how the system functions, you can use the following
1026 1026 code fragments in your programs which are ready for cut and paste:
1027 1027
1028 1028 .. literalinclude:: ../../examples/core/example-embed-short.py
1029 1029 :language: python
1030 1030
1031 1031 Using the Python debugger (pdb)
1032 1032 ===============================
1033 1033
1034 1034 Running entire programs via pdb
1035 1035 -------------------------------
1036 1036
1037 1037 pdb, the Python debugger, is a powerful interactive debugger which
1038 1038 allows you to step through code, set breakpoints, watch variables,
1039 1039 etc. IPython makes it very easy to start any script under the control
1040 1040 of pdb, regardless of whether you have wrapped it into a 'main()'
1041 1041 function or not. For this, simply type '%run -d myscript' at an
1042 1042 IPython prompt. See the %run command's documentation (via '%run?' or
1043 1043 in Sec. magic_ for more details, including how to control where pdb
1044 1044 will stop execution first.
1045 1045
1046 1046 For more information on the use of the pdb debugger, read the included
1047 1047 pdb.doc file (part of the standard Python distribution). On a stock
1048 1048 Linux system it is located at /usr/lib/python2.3/pdb.doc, but the
1049 1049 easiest way to read it is by using the help() function of the pdb module
1050 1050 as follows (in an IPython prompt)::
1051 1051
1052 1052 In [1]: import pdb
1053 1053 In [2]: pdb.help()
1054 1054
1055 1055 This will load the pdb.doc document in a file viewer for you automatically.
1056 1056
1057 1057
1058 1058 Automatic invocation of pdb on exceptions
1059 1059 -----------------------------------------
1060 1060
1061 1061 IPython, if started with the -pdb option (or if the option is set in
1062 1062 your rc file) can call the Python pdb debugger every time your code
1063 1063 triggers an uncaught exception. This feature
1064 1064 can also be toggled at any time with the %pdb magic command. This can be
1065 1065 extremely useful in order to find the origin of subtle bugs, because pdb
1066 1066 opens up at the point in your code which triggered the exception, and
1067 1067 while your program is at this point 'dead', all the data is still
1068 1068 available and you can walk up and down the stack frame and understand
1069 1069 the origin of the problem.
1070 1070
1071 1071 Furthermore, you can use these debugging facilities both with the
1072 1072 embedded IPython mode and without IPython at all. For an embedded shell
1073 1073 (see sec. Embedding_), simply call the constructor with
1074 1074 '--pdb' in the argument string and automatically pdb will be called if an
1075 1075 uncaught exception is triggered by your code.
1076 1076
1077 1077 For stand-alone use of the feature in your programs which do not use
1078 1078 IPython at all, put the following lines toward the top of your 'main'
1079 1079 routine::
1080 1080
1081 1081 import sys
1082 1082 from IPython.core import ultratb
1083 1083 sys.excepthook = ultratb.FormattedTB(mode='Verbose',
1084 1084 color_scheme='Linux', call_pdb=1)
1085 1085
1086 1086 The mode keyword can be either 'Verbose' or 'Plain', giving either very
1087 1087 detailed or normal tracebacks respectively. The color_scheme keyword can
1088 1088 be one of 'NoColor', 'Linux' (default) or 'LightBG'. These are the same
1089 1089 options which can be set in IPython with -colors and -xmode.
1090 1090
1091 1091 This will give any of your programs detailed, colored tracebacks with
1092 1092 automatic invocation of pdb.
1093 1093
1094 1094
1095 1095 Extensions for syntax processing
1096 1096 ================================
1097 1097
1098 1098 This isn't for the faint of heart, because the potential for breaking
1099 1099 things is quite high. But it can be a very powerful and useful feature.
1100 1100 In a nutshell, you can redefine the way IPython processes the user input
1101 1101 line to accept new, special extensions to the syntax without needing to
1102 1102 change any of IPython's own code.
1103 1103
1104 1104 In the IPython/extensions directory you will find some examples
1105 1105 supplied, which we will briefly describe now. These can be used 'as is'
1106 1106 (and both provide very useful functionality), or you can use them as a
1107 1107 starting point for writing your own extensions.
1108 1108
1109 1109
1110 1110 Pasting of code starting with '>>> ' or '... '
1111 1111 ----------------------------------------------
1112 1112
1113 1113 In the python tutorial it is common to find code examples which have
1114 1114 been taken from real python sessions. The problem with those is that all
1115 1115 the lines begin with either '>>> ' or '... ', which makes it impossible
1116 1116 to paste them all at once. One must instead do a line by line manual
1117 1117 copying, carefully removing the leading extraneous characters.
1118 1118
1119 1119 This extension identifies those starting characters and removes them
1120 1120 from the input automatically, so that one can paste multi-line examples
1121 1121 directly into IPython, saving a lot of time. Please look at the file
1122 1122 InterpreterPasteInput.py in the IPython/extensions directory for details
1123 1123 on how this is done.
1124 1124
1125 1125 IPython comes with a special profile enabling this feature, called
1126 1126 tutorial. Simply start IPython via 'ipython -p tutorial' and the feature
1127 1127 will be available. In a normal IPython session you can activate the
1128 1128 feature by importing the corresponding module with:
1129 1129 In [1]: import IPython.extensions.InterpreterPasteInput
1130 1130
1131 1131 The following is a 'screenshot' of how things work when this extension
1132 1132 is on, copying an example from the standard tutorial::
1133 1133
1134 1134 IPython profile: tutorial
1135 1135
1136 1136 *** Pasting of code with ">>>" or "..." has been enabled.
1137 1137
1138 1138 In [1]: >>> def fib2(n): # return Fibonacci series up to n
1139 1139 ...: ... """Return a list containing the Fibonacci series up to
1140 1140 n."""
1141 1141 ...: ... result = []
1142 1142 ...: ... a, b = 0, 1
1143 1143 ...: ... while b < n:
1144 1144 ...: ... result.append(b) # see below
1145 1145 ...: ... a, b = b, a+b
1146 1146 ...: ... return result
1147 1147 ...:
1148 1148
1149 1149 In [2]: fib2(10)
1150 1150 Out[2]: [1, 1, 2, 3, 5, 8]
1151 1151
1152 1152 Note that as currently written, this extension does not recognize
1153 1153 IPython's prompts for pasting. Those are more complicated, since the
1154 1154 user can change them very easily, they involve numbers and can vary in
1155 1155 length. One could however extract all the relevant information from the
1156 1156 IPython instance and build an appropriate regular expression. This is
1157 1157 left as an exercise for the reader.
1158 1158
1159 1159
1160 1160 Input of physical quantities with units
1161 1161 ---------------------------------------
1162 1162
1163 1163 The module PhysicalQInput allows a simplified form of input for physical
1164 1164 quantities with units. This file is meant to be used in conjunction with
1165 1165 the PhysicalQInteractive module (in the same directory) and
1166 1166 Physics.PhysicalQuantities from Konrad Hinsen's ScientificPython
1167 1167 (http://dirac.cnrs-orleans.fr/ScientificPython/).
1168 1168
1169 1169 The Physics.PhysicalQuantities module defines PhysicalQuantity objects,
1170 1170 but these must be declared as instances of a class. For example, to
1171 1171 define v as a velocity of 3 m/s, normally you would write::
1172 1172
1173 1173 In [1]: v = PhysicalQuantity(3,'m/s')
1174 1174
1175 1175 Using the PhysicalQ_Input extension this can be input instead as:
1176 1176 In [1]: v = 3 m/s
1177 1177 which is much more convenient for interactive use (even though it is
1178 1178 blatantly invalid Python syntax).
1179 1179
1180 1180 The physics profile supplied with IPython (enabled via 'ipython -p
1181 1181 physics') uses these extensions, which you can also activate with:
1182 1182
1183 1183 from math import * # math MUST be imported BEFORE PhysicalQInteractive
1184 1184 from IPython.extensions.PhysicalQInteractive import *
1185 1185 import IPython.extensions.PhysicalQInput
1186 1186
1187 1187 .. _gui_support:
1188 1188
1189 1189 GUI event loop support support
1190 1190 ==============================
1191 1191
1192 1192 .. versionadded:: 0.11
1193 1193 The ``%gui`` magic and :mod:`IPython.lib.inputhook`.
1194 1194
1195 1195 IPython has excellent support for working interactively with Graphical User
1196 1196 Interface (GUI) toolkits, such as wxPython, PyQt4, PyGTK and Tk. This is
1197 1197 implemented using Python's builtin ``PyOSInputHook`` hook. This implementation
1198 1198 is extremely robust compared to our previous threaded based version. The
1199 1199 advantages of this are:
1200 1200
1201 1201 * GUIs can be enabled and disabled dynamically at runtime.
1202 1202 * The active GUI can be switched dynamically at runtime.
1203 1203 * In some cases, multiple GUIs can run simultaneously with no problems.
1204 1204 * There is a developer API in :mod:`IPython.lib.inputhook` for customizing
1205 1205 all of these things.
1206 1206
1207 1207 For users, enabling GUI event loop integration is simple. You simple use the
1208 1208 ``%gui`` magic as follows::
1209 1209
1210 1210 %gui [-a] [GUINAME]
1211 1211
1212 1212 With no arguments, ``%gui`` removes all GUI support. Valid ``GUINAME``
1213 1213 arguments are ``wx``, ``qt4``, ``gtk`` and ``tk``. The ``-a`` option will
1214 1214 create and return a running application object for the selected GUI toolkit.
1215 1215
1216 1216 Thus, to use wxPython interactively and create a running :class:`wx.App`
1217 1217 object, do::
1218 1218
1219 1219 %gui -a wx
1220 1220
1221 1221 For information on IPython's Matplotlib integration (and the ``pylab`` mode)
1222 1222 see :ref:`this section <matplotlib_support>`.
1223 1223
1224 1224 For developers that want to use IPython's GUI event loop integration in
1225 1225 the form of a library, these capabilities are exposed in library form
1226 1226 in the :mod:`IPython.lib.inputhook`. Interested developers should see the
1227 1227 module docstrings for more information, but there are a few points that
1228 1228 should be mentioned here.
1229 1229
1230 1230 First, the ``PyOSInputHook`` approach only works in command line settings
1231 1231 where readline is activated.
1232 1232
1233 1233 Second, when using the ``PyOSInputHook`` approach, a GUI application should
1234 1234 *not* start its event loop. Instead all of this is handled by the
1235 1235 ``PyOSInputHook``. This means that applications that are meant to be used both
1236 1236 in IPython and as standalone apps need to have special code to detects how the
1237 1237 application is being run. We highly recommend using IPython's
1238 1238 :func:`enable_foo` functions for this. Here is a simple example that shows the
1239 1239 recommended code that should be at the bottom of a wxPython using GUI
1240 1240 application::
1241 1241
1242 1242 try:
1243 1243 from IPython.lib.inputhook import enable_wx
1244 1244 enable_wx(app)
1245 1245 except ImportError:
1246 1246 app.MainLoop()
1247 1247
1248 1248 This pattern should be used instead of the simple ``app.MainLoop()`` code
1249 1249 that a standalone wxPython application would have.
1250 1250
1251 1251 Third, unlike previous versions of IPython, we no longer "hijack" (replace
1252 1252 them with no-ops) the event loops. This is done to allow applications that
1253 1253 actually need to run the real event loops to do so. This is often needed to
1254 1254 process pending events at critical points.
1255 1255
1256 1256 Finally, we also have a number of examples in our source directory
1257 1257 :file:`docs/examples/lib` that demonstrate these capabilities.
1258 1258
1259 1259 PyQt and PySide
1260 1260 ---------------
1261 1261
1262 1262 .. attempt at explanation of the complete mess that is Qt support
1263 1263
1264 1264 When you use ``gui=qt`` or ``pylab=qt``, IPython can work with either
1265 1265 PyQt4 or PySide. There are three options for configuration here, because
1266 1266 PyQt4 has two APIs for QString and QVariant - v1, which is the default on
1267 1267 Python 2, and the more natural v2, which is the only API supported by PySide.
1268 1268 v2 is also the default for PyQt4 on Python 3. IPython's code for the QtConsole
1269 1269 uses v2, but you can still use any interface in your code, since the
1270 1270 Qt frontend is in a different process.
1271 1271
1272 1272 The default will be to import PyQt4 without configuration of the APIs, thus
1273 1273 matching what most applications would expect. It will fall back of PySide if
1274 1274 PyQt4 is unavailable.
1275 1275
1276 1276 If specified, IPython will respect the environment variable ``QT_API`` used
1277 1277 by ETS. ETS 4.0 also works with both PyQt4 and PySide, but it requires
1278 1278 PyQt4 to use its v2 API. So if ``QT_API=pyside`` PySide will be used,
1279 1279 and if ``QT_API=pyqt`` then PyQt4 will be used *with the v2 API* for
1280 1280 QString and QVariant, so ETS codes like MayaVi will also work with IPython.
1281 1281
1282 1282 If you launch IPython in pylab mode with ``ipython pylab=qt``, then IPython
1283 1283 will ask matplotlib which Qt library to use (only if QT_API is *not set*),
1284 1284 via the 'backend.qt4' rcParam.
1285 1285 If matplotlib is version 1.0.1 or older, then IPython will always use PyQt4
1286 1286 without setting the v2 APIs, since neither v2 PyQt nor PySide work.
1287 1287
1288 1288 .. warning::
1289 1289
1290 1290 Note that this means for ETS 4 to work with PyQt4, ``QT_API`` *must* be set to
1291 1291 work with IPython's qt integration, because otherwise PyQt4 will be loaded in
1292 1292 an incompatible mode.
1293 1293
1294 1294 It also means that you must *not* have ``QT_API`` set if you want to
1295 1295 use ``gui=qt`` with code that requires PyQt4 API v1.
1296 1296
1297 1297
1298 1298
1299 1299 .. _matplotlib_support:
1300 1300
1301 1301 Plotting with matplotlib
1302 1302 ========================
1303 1303
1304 1304
1305 1305 `Matplotlib`_ provides high quality 2D and
1306 1306 3D plotting for Python. Matplotlib can produce plots on screen using a variety
1307 1307 of GUI toolkits, including Tk, PyGTK, PyQt4 and wxPython. It also provides a
1308 1308 number of commands useful for scientific computing, all with a syntax
1309 1309 compatible with that of the popular Matlab program.
1310 1310
1311 1311 Many IPython users have come to rely on IPython's ``-pylab`` mode which
1312 1312 automates the integration of Matplotlib with IPython. We are still in the
1313 1313 process of working with the Matplotlib developers to finalize the new pylab
1314 1314 API, but for now you can use Matplotlib interactively using the following
1315 1315 commands::
1316 1316
1317 1317 %gui -a wx
1318 1318 import matplotlib
1319 1319 matplotlib.use('wxagg')
1320 1320 from matplotlib import pylab
1321 1321 pylab.interactive(True)
1322 1322
1323 1323 All of this will soon be automated as Matplotlib begins to include
1324 1324 new logic that uses our new GUI support.
1325 1325
1326 1326 .. _Matplotlib: http://matplotlib.sourceforge.net
1327 1327
1328 1328 .. _interactive_demos:
1329 1329
1330 1330 Interactive demos with IPython
1331 1331 ==============================
1332 1332
1333 1333 IPython ships with a basic system for running scripts interactively in
1334 1334 sections, useful when presenting code to audiences. A few tags embedded
1335 1335 in comments (so that the script remains valid Python code) divide a file
1336 1336 into separate blocks, and the demo can be run one block at a time, with
1337 1337 IPython printing (with syntax highlighting) the block before executing
1338 1338 it, and returning to the interactive prompt after each block. The
1339 1339 interactive namespace is updated after each block is run with the
1340 1340 contents of the demo's namespace.
1341 1341
1342 1342 This allows you to show a piece of code, run it and then execute
1343 1343 interactively commands based on the variables just created. Once you
1344 1344 want to continue, you simply execute the next block of the demo. The
1345 1345 following listing shows the markup necessary for dividing a script into
1346 1346 sections for execution as a demo:
1347 1347
1348 1348 .. literalinclude:: ../../examples/lib/example-demo.py
1349 1349 :language: python
1350 1350
1351 1351
1352 1352 In order to run a file as a demo, you must first make a Demo object out
1353 1353 of it. If the file is named myscript.py, the following code will make a
1354 1354 demo::
1355 1355
1356 1356 from IPython.lib.demo import Demo
1357 1357
1358 1358 mydemo = Demo('myscript.py')
1359 1359
1360 1360 This creates the mydemo object, whose blocks you run one at a time by
1361 1361 simply calling the object with no arguments. If you have autocall active
1362 1362 in IPython (the default), all you need to do is type::
1363 1363
1364 1364 mydemo
1365 1365
1366 1366 and IPython will call it, executing each block. Demo objects can be
1367 1367 restarted, you can move forward or back skipping blocks, re-execute the
1368 1368 last block, etc. Simply use the Tab key on a demo object to see its
1369 1369 methods, and call '?' on them to see their docstrings for more usage
1370 1370 details. In addition, the demo module itself contains a comprehensive
1371 1371 docstring, which you can access via::
1372 1372
1373 1373 from IPython.lib import demo
1374 1374
1375 1375 demo?
1376 1376
1377 1377 Limitations: It is important to note that these demos are limited to
1378 1378 fairly simple uses. In particular, you can not put division marks in
1379 1379 indented code (loops, if statements, function definitions, etc.)
1380 1380 Supporting something like this would basically require tracking the
1381 1381 internal execution state of the Python interpreter, so only top-level
1382 1382 divisions are allowed. If you want to be able to open an IPython
1383 1383 instance at an arbitrary point in a program, you can use IPython's
1384 1384 embedding facilities, described in detail in Sec. 9
1385 1385
@@ -1,691 +1,691 b''
1 1 .. _parallel_process:
2 2
3 3 ===========================================
4 4 Starting the IPython controller and engines
5 5 ===========================================
6 6
7 7 To use IPython for parallel computing, you need to start one instance of
8 8 the controller and one or more instances of the engine. The controller
9 9 and each engine can run on different machines or on the same machine.
10 10 Because of this, there are many different possibilities.
11 11
12 12 Broadly speaking, there are two ways of going about starting a controller and engines:
13 13
14 14 * In an automated manner using the :command:`ipcluster` command.
15 15 * In a more manual way using the :command:`ipcontroller` and
16 16 :command:`ipengine` commands.
17 17
18 18 This document describes both of these methods. We recommend that new users
19 19 start with the :command:`ipcluster` command as it simplifies many common usage
20 20 cases.
21 21
22 22 General considerations
23 23 ======================
24 24
25 25 Before delving into the details about how you can start a controller and
26 26 engines using the various methods, we outline some of the general issues that
27 27 come up when starting the controller and engines. These things come up no
28 28 matter which method you use to start your IPython cluster.
29 29
30 30 If you are running engines on multiple machines, you will likely need to instruct the
31 31 controller to listen for connections on an external interface. This can be done by specifying
32 32 the ``ip`` argument on the command-line, or the ``HubFactory.ip`` configurable in
33 33 :file:`ipcontroller_config.py`.
34 34
35 35 If your machines are on a trusted network, you can safely instruct the controller to listen
36 36 on all public interfaces with::
37 37
38 38 $> ipcontroller --ip=*
39 39
40 40 Or you can set the same behavior as the default by adding the following line to your :file:`ipcontroller_config.py`:
41 41
42 42 .. sourcecode:: python
43 43
44 44 c.HubFactory.ip = '*'
45 45
46 46 .. note::
47 47
48 48 Due to the lack of security in ZeroMQ, the controller will only listen for connections on
49 49 localhost by default. If you see Timeout errors on engines or clients, then the first
50 50 thing you should check is the ip address the controller is listening on, and make sure
51 51 that it is visible from the timing out machine.
52 52
53 53 .. seealso::
54 54
55 55 Our `notes <parallel_security>`_ on security in the new parallel computing code.
56 56
57 57 Let's say that you want to start the controller on ``host0`` and engines on
58 58 hosts ``host1``-``hostn``. The following steps are then required:
59 59
60 60 1. Start the controller on ``host0`` by running :command:`ipcontroller` on
61 61 ``host0``. The controller must be instructed to listen on an interface visible
62 62 to the engine machines, via the ``ip`` command-line argument or ``HubFactory.ip``
63 63 in :file:`ipcontroller_config.py`.
64 64 2. Move the JSON file (:file:`ipcontroller-engine.json`) created by the
65 65 controller from ``host0`` to hosts ``host1``-``hostn``.
66 66 3. Start the engines on hosts ``host1``-``hostn`` by running
67 67 :command:`ipengine`. This command has to be told where the JSON file
68 68 (:file:`ipcontroller-engine.json`) is located.
69 69
70 70 At this point, the controller and engines will be connected. By default, the JSON files
71 71 created by the controller are put into the :file:`~/.ipython/profile_default/security`
72 72 directory. If the engines share a filesystem with the controller, step 2 can be skipped as
73 73 the engines will automatically look at that location.
74 74
75 75 The final step required to actually use the running controller from a client is to move
76 76 the JSON file :file:`ipcontroller-client.json` from ``host0`` to any host where clients
77 77 will be run. If these file are put into the :file:`~/.ipython/profile_default/security`
78 78 directory of the client's host, they will be found automatically. Otherwise, the full path
79 79 to them has to be passed to the client's constructor.
80 80
81 81 Using :command:`ipcluster`
82 82 ===========================
83 83
84 84 The :command:`ipcluster` command provides a simple way of starting a
85 85 controller and engines in the following situations:
86 86
87 87 1. When the controller and engines are all run on localhost. This is useful
88 88 for testing or running on a multicore computer.
89 89 2. When engines are started using the :command:`mpiexec` command that comes
90 90 with most MPI [MPI]_ implementations
91 91 3. When engines are started using the PBS [PBS]_ batch system
92 92 (or other `qsub` systems, such as SGE).
93 93 4. When the controller is started on localhost and the engines are started on
94 94 remote nodes using :command:`ssh`.
95 95 5. When engines are started using the Windows HPC Server batch system.
96 96
97 97 .. note::
98 98
99 99 Currently :command:`ipcluster` requires that the
100 100 :file:`~/.ipython/profile_<name>/security` directory live on a shared filesystem that is
101 101 seen by both the controller and engines. If you don't have a shared file
102 102 system you will need to use :command:`ipcontroller` and
103 103 :command:`ipengine` directly.
104 104
105 105 Under the hood, :command:`ipcluster` just uses :command:`ipcontroller`
106 106 and :command:`ipengine` to perform the steps described above.
107 107
108 108 The simplest way to use ipcluster requires no configuration, and will
109 109 launch a controller and a number of engines on the local machine. For instance,
110 110 to start one controller and 4 engines on localhost, just do::
111 111
112 112 $ ipcluster start --n=4
113 113
114 114 To see other command line options, do::
115 115
116 116 $ ipcluster -h
117 117
118 118
119 119 Configuring an IPython cluster
120 120 ==============================
121 121
122 122 Cluster configurations are stored as `profiles`. You can create a new profile with::
123 123
124 124 $ ipython profile create --parallel --profile=myprofile
125 125
126 126 This will create the directory :file:`IPYTHONDIR/profile_myprofile`, and populate it
127 127 with the default configuration files for the three IPython cluster commands. Once
128 128 you edit those files, you can continue to call ipcluster/ipcontroller/ipengine
129 129 with no arguments beyond ``profile=myprofile``, and any configuration will be maintained.
130 130
131 131 There is no limit to the number of profiles you can have, so you can maintain a profile for each
132 132 of your common use cases. The default profile will be used whenever the
133 133 profile argument is not specified, so edit :file:`IPYTHONDIR/profile_default/*_config.py` to
134 134 represent your most common use case.
135 135
136 136 The configuration files are loaded with commented-out settings and explanations,
137 137 which should cover most of the available possibilities.
138 138
139 139 Using various batch systems with :command:`ipcluster`
140 140 -----------------------------------------------------
141 141
142 142 :command:`ipcluster` has a notion of Launchers that can start controllers
143 143 and engines with various remote execution schemes. Currently supported
144 144 models include :command:`ssh`, :command:`mpiexec`, PBS-style (Torque, SGE),
145 145 and Windows HPC Server.
146 146
147 147 .. note::
148 148
149 149 The Launchers and configuration are designed in such a way that advanced
150 150 users can subclass and configure them to fit their own system that we
151 151 have not yet supported (such as Condor)
152 152
153 153 Using :command:`ipcluster` in mpiexec/mpirun mode
154 154 --------------------------------------------------
155 155
156 156
157 157 The mpiexec/mpirun mode is useful if you:
158 158
159 159 1. Have MPI installed.
160 160 2. Your systems are configured to use the :command:`mpiexec` or
161 161 :command:`mpirun` commands to start MPI processes.
162 162
163 163 If these are satisfied, you can create a new profile::
164 164
165 165 $ ipython profile create --parallel --profile=mpi
166 166
167 167 and edit the file :file:`IPYTHONDIR/profile_mpi/ipcluster_config.py`.
168 168
169 169 There, instruct ipcluster to use the MPIExec launchers by adding the lines:
170 170
171 171 .. sourcecode:: python
172 172
173 173 c.IPClusterEngines.engine_launcher = 'IPython.parallel.apps.launcher.MPIExecEngineSetLauncher'
174 174
175 175 If the default MPI configuration is correct, then you can now start your cluster, with::
176 176
177 177 $ ipcluster start --n=4 --profile=mpi
178 178
179 179 This does the following:
180 180
181 181 1. Starts the IPython controller on current host.
182 182 2. Uses :command:`mpiexec` to start 4 engines.
183 183
184 184 If you have a reason to also start the Controller with mpi, you can specify:
185 185
186 186 .. sourcecode:: python
187 187
188 188 c.IPClusterStart.controller_launcher = 'IPython.parallel.apps.launcher.MPIExecControllerLauncher'
189 189
190 190 .. note::
191 191
192 192 The Controller *will not* be in the same MPI universe as the engines, so there is not
193 193 much reason to do this unless sysadmins demand it.
194 194
195 195 On newer MPI implementations (such as OpenMPI), this will work even if you
196 196 don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI
197 197 implementations actually require each process to call :func:`MPI_Init` upon
198 198 starting. The easiest way of having this done is to install the mpi4py
199 199 [mpi4py]_ package and then specify the ``c.MPI.use`` option in :file:`ipengine_config.py`:
200 200
201 201 .. sourcecode:: python
202 202
203 203 c.MPI.use = 'mpi4py'
204 204
205 205 Unfortunately, even this won't work for some MPI implementations. If you are
206 206 having problems with this, you will likely have to use a custom Python
207 207 executable that itself calls :func:`MPI_Init` at the appropriate time.
208 208 Fortunately, mpi4py comes with such a custom Python executable that is easy to
209 209 install and use. However, this custom Python executable approach will not work
210 210 with :command:`ipcluster` currently.
211 211
212 212 More details on using MPI with IPython can be found :ref:`here <parallelmpi>`.
213 213
214 214
215 215 Using :command:`ipcluster` in PBS mode
216 216 ---------------------------------------
217 217
218 218 The PBS mode uses the Portable Batch System (PBS) to start the engines.
219 219
220 220 As usual, we will start by creating a fresh profile::
221 221
222 222 $ ipython profile create --parallel --profile=pbs
223 223
224 224 And in :file:`ipcluster_config.py`, we will select the PBS launchers for the controller
225 225 and engines:
226 226
227 227 .. sourcecode:: python
228 228
229 229 c.IPClusterStart.controller_launcher = \
230 230 'IPython.parallel.apps.launcher.PBSControllerLauncher'
231 231 c.IPClusterEngines.engine_launcher = \
232 232 'IPython.parallel.apps.launcher.PBSEngineSetLauncher'
233 233
234 234 .. note::
235 235
236 236 Note that the configurable is IPClusterEngines for the engine launcher, and
237 237 IPClusterStart for the controller launcher. This is because the start command is a
238 238 subclass of the engine command, adding a controller launcher. Since it is a subclass,
239 239 any configuration made in IPClusterEngines is inherited by IPClusterStart unless it is
240 240 overridden.
241 241
242 242 IPython does provide simple default batch templates for PBS and SGE, but you may need
243 243 to specify your own. Here is a sample PBS script template:
244 244
245 245 .. sourcecode:: bash
246 246
247 247 #PBS -N ipython
248 248 #PBS -j oe
249 249 #PBS -l walltime=00:10:00
250 250 #PBS -l nodes={n/4}:ppn=4
251 251 #PBS -q {queue}
252 252
253 253 cd $PBS_O_WORKDIR
254 254 export PATH=$HOME/usr/local/bin
255 255 export PYTHONPATH=$HOME/usr/local/lib/python2.7/site-packages
256 /usr/local/bin/mpiexec -n {n} ipengine --profile_dir={profile_dir}
256 /usr/local/bin/mpiexec -n {n} ipengine --profile-dir={profile_dir}
257 257
258 258 There are a few important points about this template:
259 259
260 260 1. This template will be rendered at runtime using IPython's :class:`EvalFormatter`.
261 261 This is simply a subclass of :class:`string.Formatter` that allows simple expressions
262 262 on keys.
263 263
264 264 2. Instead of putting in the actual number of engines, use the notation
265 265 ``{n}`` to indicate the number of engines to be started. You can also use
266 266 expressions like ``{n/4}`` in the template to indicate the number of nodes.
267 267 There will always be ``{n}`` and ``{profile_dir}`` variables passed to the formatter.
268 268 These allow the batch system to know how many engines, and where the configuration
269 269 files reside. The same is true for the batch queue, with the template variable
270 270 ``{queue}``.
271 271
272 272 3. Any options to :command:`ipengine` can be given in the batch script
273 273 template, or in :file:`ipengine_config.py`.
274 274
275 275 4. Depending on the configuration of you system, you may have to set
276 276 environment variables in the script template.
277 277
278 278 The controller template should be similar, but simpler:
279 279
280 280 .. sourcecode:: bash
281 281
282 282 #PBS -N ipython
283 283 #PBS -j oe
284 284 #PBS -l walltime=00:10:00
285 285 #PBS -l nodes=1:ppn=4
286 286 #PBS -q {queue}
287 287
288 288 cd $PBS_O_WORKDIR
289 289 export PATH=$HOME/usr/local/bin
290 290 export PYTHONPATH=$HOME/usr/local/lib/python2.7/site-packages
291 ipcontroller --profile_dir={profile_dir}
291 ipcontroller --profile-dir={profile_dir}
292 292
293 293
294 294 Once you have created these scripts, save them with names like
295 295 :file:`pbs.engine.template`. Now you can load them into the :file:`ipcluster_config` with:
296 296
297 297 .. sourcecode:: python
298 298
299 299 c.PBSEngineSetLauncher.batch_template_file = "pbs.engine.template"
300 300
301 301 c.PBSControllerLauncher.batch_template_file = "pbs.controller.template"
302 302
303 303
304 304 Alternately, you can just define the templates as strings inside :file:`ipcluster_config`.
305 305
306 306 Whether you are using your own templates or our defaults, the extra configurables available are
307 307 the number of engines to launch (``{n}``, and the batch system queue to which the jobs are to be
308 308 submitted (``{queue}``)). These are configurables, and can be specified in
309 309 :file:`ipcluster_config`:
310 310
311 311 .. sourcecode:: python
312 312
313 313 c.PBSLauncher.queue = 'veryshort.q'
314 314 c.IPClusterEngines.n = 64
315 315
316 316 Note that assuming you are running PBS on a multi-node cluster, the Controller's default behavior
317 317 of listening only on localhost is likely too restrictive. In this case, also assuming the
318 318 nodes are safely behind a firewall, you can simply instruct the Controller to listen for
319 319 connections on all its interfaces, by adding in :file:`ipcontroller_config`:
320 320
321 321 .. sourcecode:: python
322 322
323 323 c.HubFactory.ip = '*'
324 324
325 325 You can now run the cluster with::
326 326
327 327 $ ipcluster start --profile=pbs --n=128
328 328
329 329 Additional configuration options can be found in the PBS section of :file:`ipcluster_config`.
330 330
331 331 .. note::
332 332
333 333 Due to the flexibility of configuration, the PBS launchers work with simple changes
334 334 to the template for other :command:`qsub`-using systems, such as Sun Grid Engine,
335 335 and with further configuration in similar batch systems like Condor.
336 336
337 337
338 338 Using :command:`ipcluster` in SSH mode
339 339 ---------------------------------------
340 340
341 341
342 342 The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote
343 343 nodes and :command:`ipcontroller` can be run remotely as well, or on localhost.
344 344
345 345 .. note::
346 346
347 347 When using this mode it highly recommended that you have set up SSH keys
348 348 and are using ssh-agent [SSH]_ for password-less logins.
349 349
350 350 As usual, we start by creating a clean profile::
351 351
352 352 $ ipython profile create --parallel --profile=ssh
353 353
354 354 To use this mode, select the SSH launchers in :file:`ipcluster_config.py`:
355 355
356 356 .. sourcecode:: python
357 357
358 358 c.IPClusterEngines.engine_launcher = \
359 359 'IPython.parallel.apps.launcher.SSHEngineSetLauncher'
360 360 # and if the Controller is also to be remote:
361 361 c.IPClusterStart.controller_launcher = \
362 362 'IPython.parallel.apps.launcher.SSHControllerLauncher'
363 363
364 364
365 365 The controller's remote location and configuration can be specified:
366 366
367 367 .. sourcecode:: python
368 368
369 369 # Set the user and hostname for the controller
370 370 # c.SSHControllerLauncher.hostname = 'controller.example.com'
371 371 # c.SSHControllerLauncher.user = os.environ.get('USER','username')
372 372
373 373 # Set the arguments to be passed to ipcontroller
374 374 # note that remotely launched ipcontroller will not get the contents of
375 375 # the local ipcontroller_config.py unless it resides on the *remote host*
376 # in the location specified by the `profile_dir` argument.
377 # c.SSHControllerLauncher.program_args = ['--reuse', '--ip=*', '--profile_dir=/path/to/cd']
376 # in the location specified by the `profile-dir` argument.
377 # c.SSHControllerLauncher.program_args = ['--reuse', '--ip=*', '--profile-dir=/path/to/cd']
378 378
379 379 .. note::
380 380
381 381 SSH mode does not do any file movement, so you will need to distribute configuration
382 382 files manually. To aid in this, the `reuse_files` flag defaults to True for ssh-launched
383 383 Controllers, so you will only need to do this once, unless you override this flag back
384 384 to False.
385 385
386 386 Engines are specified in a dictionary, by hostname and the number of engines to be run
387 387 on that host.
388 388
389 389 .. sourcecode:: python
390 390
391 391 c.SSHEngineSetLauncher.engines = { 'host1.example.com' : 2,
392 392 'host2.example.com' : 5,
393 'host3.example.com' : (1, ['--profile_dir=/home/different/location']),
393 'host3.example.com' : (1, ['--profile-dir=/home/different/location']),
394 394 'host4.example.com' : 8 }
395 395
396 396 * The `engines` dict, where the keys are the host we want to run engines on and
397 397 the value is the number of engines to run on that host.
398 398 * on host3, the value is a tuple, where the number of engines is first, and the arguments
399 399 to be passed to :command:`ipengine` are the second element.
400 400
401 401 For engines without explicitly specified arguments, the default arguments are set in
402 402 a single location:
403 403
404 404 .. sourcecode:: python
405 405
406 c.SSHEngineSetLauncher.engine_args = ['--profile_dir=/path/to/profile_ssh']
406 c.SSHEngineSetLauncher.engine_args = ['--profile-dir=/path/to/profile_ssh']
407 407
408 408 Current limitations of the SSH mode of :command:`ipcluster` are:
409 409
410 410 * Untested on Windows. Would require a working :command:`ssh` on Windows.
411 411 Also, we are using shell scripts to setup and execute commands on remote
412 412 hosts.
413 413 * No file movement - This is a regression from 0.10, which moved connection files
414 414 around with scp. This will be improved, but not before 0.11 release.
415 415
416 416 Using the :command:`ipcontroller` and :command:`ipengine` commands
417 417 ====================================================================
418 418
419 419 It is also possible to use the :command:`ipcontroller` and :command:`ipengine`
420 420 commands to start your controller and engines. This approach gives you full
421 421 control over all aspects of the startup process.
422 422
423 423 Starting the controller and engine on your local machine
424 424 --------------------------------------------------------
425 425
426 426 To use :command:`ipcontroller` and :command:`ipengine` to start things on your
427 427 local machine, do the following.
428 428
429 429 First start the controller::
430 430
431 431 $ ipcontroller
432 432
433 433 Next, start however many instances of the engine you want using (repeatedly)
434 434 the command::
435 435
436 436 $ ipengine
437 437
438 438 The engines should start and automatically connect to the controller using the
439 439 JSON files in :file:`~/.ipython/profile_default/security`. You are now ready to use the
440 440 controller and engines from IPython.
441 441
442 442 .. warning::
443 443
444 444 The order of the above operations may be important. You *must*
445 445 start the controller before the engines, unless you are reusing connection
446 446 information (via ``--reuse``), in which case ordering is not important.
447 447
448 448 .. note::
449 449
450 450 On some platforms (OS X), to put the controller and engine into the
451 451 background you may need to give these commands in the form ``(ipcontroller
452 452 &)`` and ``(ipengine &)`` (with the parentheses) for them to work
453 453 properly.
454 454
455 455 Starting the controller and engines on different hosts
456 456 ------------------------------------------------------
457 457
458 458 When the controller and engines are running on different hosts, things are
459 459 slightly more complicated, but the underlying ideas are the same:
460 460
461 461 1. Start the controller on a host using :command:`ipcontroller`. The controller must be
462 462 instructed to listen on an interface visible to the engine machines, via the ``ip``
463 463 command-line argument or ``HubFactory.ip`` in :file:`ipcontroller_config.py`.
464 464 2. Copy :file:`ipcontroller-engine.json` from :file:`~/.ipython/profile_<name>/security` on
465 465 the controller's host to the host where the engines will run.
466 466 3. Use :command:`ipengine` on the engine's hosts to start the engines.
467 467
468 468 The only thing you have to be careful of is to tell :command:`ipengine` where
469 469 the :file:`ipcontroller-engine.json` file is located. There are two ways you
470 470 can do this:
471 471
472 472 * Put :file:`ipcontroller-engine.json` in the :file:`~/.ipython/profile_<name>/security`
473 473 directory on the engine's host, where it will be found automatically.
474 474 * Call :command:`ipengine` with the ``--file=full_path_to_the_file``
475 475 flag.
476 476
477 477 The ``file`` flag works like this::
478 478
479 479 $ ipengine --file=/path/to/my/ipcontroller-engine.json
480 480
481 481 .. note::
482 482
483 483 If the controller's and engine's hosts all have a shared file system
484 484 (:file:`~/.ipython/profile_<name>/security` is the same on all of them), then things
485 485 will just work!
486 486
487 487 Make JSON files persistent
488 488 --------------------------
489 489
490 490 At fist glance it may seem that that managing the JSON files is a bit
491 491 annoying. Going back to the house and key analogy, copying the JSON around
492 492 each time you start the controller is like having to make a new key every time
493 493 you want to unlock the door and enter your house. As with your house, you want
494 494 to be able to create the key (or JSON file) once, and then simply use it at
495 495 any point in the future.
496 496
497 497 To do this, the only thing you have to do is specify the `--reuse` flag, so that
498 498 the connection information in the JSON files remains accurate::
499 499
500 500 $ ipcontroller --reuse
501 501
502 502 Then, just copy the JSON files over the first time and you are set. You can
503 503 start and stop the controller and engines any many times as you want in the
504 504 future, just make sure to tell the controller to reuse the file.
505 505
506 506 .. note::
507 507
508 508 You may ask the question: what ports does the controller listen on if you
509 509 don't tell is to use specific ones? The default is to use high random port
510 510 numbers. We do this for two reasons: i) to increase security through
511 511 obscurity and ii) to multiple controllers on a given host to start and
512 512 automatically use different ports.
513 513
514 514 Log files
515 515 ---------
516 516
517 517 All of the components of IPython have log files associated with them.
518 518 These log files can be extremely useful in debugging problems with
519 519 IPython and can be found in the directory :file:`~/.ipython/profile_<name>/log`.
520 520 Sending the log files to us will often help us to debug any problems.
521 521
522 522
523 523 Configuring `ipcontroller`
524 524 ---------------------------
525 525
526 526 The IPython Controller takes its configuration from the file :file:`ipcontroller_config.py`
527 527 in the active profile directory.
528 528
529 529 Ports and addresses
530 530 *******************
531 531
532 532 In many cases, you will want to configure the Controller's network identity. By default,
533 533 the Controller listens only on loopback, which is the most secure but often impractical.
534 534 To instruct the controller to listen on a specific interface, you can set the
535 535 :attr:`HubFactory.ip` trait. To listen on all interfaces, simply specify:
536 536
537 537 .. sourcecode:: python
538 538
539 539 c.HubFactory.ip = '*'
540 540
541 541 When connecting to a Controller that is listening on loopback or behind a firewall, it may
542 542 be necessary to specify an SSH server to use for tunnels, and the external IP of the
543 543 Controller. If you specified that the HubFactory listen on loopback, or all interfaces,
544 544 then IPython will try to guess the external IP. If you are on a system with VM network
545 545 devices, or many interfaces, this guess may be incorrect. In these cases, you will want
546 546 to specify the 'location' of the Controller. This is the IP of the machine the Controller
547 547 is on, as seen by the clients, engines, or the SSH server used to tunnel connections.
548 548
549 549 For example, to set up a cluster with a Controller on a work node, using ssh tunnels
550 550 through the login node, an example :file:`ipcontroller_config.py` might contain:
551 551
552 552 .. sourcecode:: python
553 553
554 554 # allow connections on all interfaces from engines
555 555 # engines on the same node will use loopback, while engines
556 556 # from other nodes will use an external IP
557 557 c.HubFactory.ip = '*'
558 558
559 559 # you typically only need to specify the location when there are extra
560 560 # interfaces that may not be visible to peer nodes (e.g. VM interfaces)
561 561 c.HubFactory.location = '10.0.1.5'
562 562 # or to get an automatic value, try this:
563 563 import socket
564 564 ex_ip = socket.gethostbyname_ex(socket.gethostname())[-1][0]
565 565 c.HubFactory.location = ex_ip
566 566
567 567 # now instruct clients to use the login node for SSH tunnels:
568 568 c.HubFactory.ssh_server = 'login.mycluster.net'
569 569
570 570 After doing this, your :file:`ipcontroller-client.json` file will look something like this:
571 571
572 572 .. this can be Python, despite the fact that it's actually JSON, because it's
573 573 .. still valid Python
574 574
575 575 .. sourcecode:: python
576 576
577 577 {
578 578 "url":"tcp:\/\/*:43447",
579 579 "exec_key":"9c7779e4-d08a-4c3b-ba8e-db1f80b562c1",
580 580 "ssh":"login.mycluster.net",
581 581 "location":"10.0.1.5"
582 582 }
583 583
584 584 Then this file will be all you need for a client to connect to the controller, tunneling
585 585 SSH connections through login.mycluster.net.
586 586
587 587 Database Backend
588 588 ****************
589 589
590 590 The Hub stores all messages and results passed between Clients and Engines.
591 591 For large and/or long-running clusters, it would be unreasonable to keep all
592 592 of this information in memory. For this reason, we have two database backends:
593 593 [MongoDB]_ via PyMongo_, and SQLite with the stdlib :py:mod:`sqlite`.
594 594
595 595 MongoDB is our design target, and the dict-like model it uses has driven our design. As far
596 596 as we are concerned, BSON can be considered essentially the same as JSON, adding support
597 597 for binary data and datetime objects, and any new database backend must support the same
598 598 data types.
599 599
600 600 .. seealso::
601 601
602 602 MongoDB `BSON doc <http://www.mongodb.org/display/DOCS/BSON>`_
603 603
604 604 To use one of these backends, you must set the :attr:`HubFactory.db_class` trait:
605 605
606 606 .. sourcecode:: python
607 607
608 608 # for a simple dict-based in-memory implementation, use dictdb
609 609 # This is the default and the fastest, since it doesn't involve the filesystem
610 610 c.HubFactory.db_class = 'IPython.parallel.controller.dictdb.DictDB'
611 611
612 612 # To use MongoDB:
613 613 c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
614 614
615 615 # and SQLite:
616 616 c.HubFactory.db_class = 'IPython.parallel.controller.sqlitedb.SQLiteDB'
617 617
618 618 When using the proper databases, you can actually allow for tasks to persist from
619 619 one session to the next by specifying the MongoDB database or SQLite table in
620 620 which tasks are to be stored. The default is to use a table named for the Hub's Session,
621 621 which is a UUID, and thus different every time.
622 622
623 623 .. sourcecode:: python
624 624
625 625 # To keep persistant task history in MongoDB:
626 626 c.MongoDB.database = 'tasks'
627 627
628 628 # and in SQLite:
629 629 c.SQLiteDB.table = 'tasks'
630 630
631 631
632 632 Since MongoDB servers can be running remotely or configured to listen on a particular port,
633 633 you can specify any arguments you may need to the PyMongo `Connection
634 634 <http://api.mongodb.org/python/1.9/api/pymongo/connection.html#pymongo.connection.Connection>`_:
635 635
636 636 .. sourcecode:: python
637 637
638 638 # positional args to pymongo.Connection
639 639 c.MongoDB.connection_args = []
640 640
641 641 # keyword args to pymongo.Connection
642 642 c.MongoDB.connection_kwargs = {}
643 643
644 644 .. _MongoDB: http://www.mongodb.org
645 645 .. _PyMongo: http://api.mongodb.org/python/1.9/
646 646
647 647 Configuring `ipengine`
648 648 -----------------------
649 649
650 650 The IPython Engine takes its configuration from the file :file:`ipengine_config.py`
651 651
652 652 The Engine itself also has some amount of configuration. Most of this
653 653 has to do with initializing MPI or connecting to the controller.
654 654
655 655 To instruct the Engine to initialize with an MPI environment set up by
656 656 mpi4py, add:
657 657
658 658 .. sourcecode:: python
659 659
660 660 c.MPI.use = 'mpi4py'
661 661
662 662 In this case, the Engine will use our default mpi4py init script to set up
663 663 the MPI environment prior to exection. We have default init scripts for
664 664 mpi4py and pytrilinos. If you want to specify your own code to be run
665 665 at the beginning, specify `c.MPI.init_script`.
666 666
667 667 You can also specify a file or python command to be run at startup of the
668 668 Engine:
669 669
670 670 .. sourcecode:: python
671 671
672 672 c.IPEngineApp.startup_script = u'/path/to/my/startup.py'
673 673
674 674 c.IPEngineApp.startup_command = 'import numpy, scipy, mpi4py'
675 675
676 676 These commands/files will be run again, after each
677 677
678 678 It's also useful on systems with shared filesystems to run the engines
679 679 in some scratch directory. This can be set with:
680 680
681 681 .. sourcecode:: python
682 682
683 683 c.IPEngineApp.work_dir = u'/path/to/scratch/'
684 684
685 685
686 686
687 687 .. [MongoDB] MongoDB database http://www.mongodb.org
688 688
689 689 .. [PBS] Portable Batch System http://www.openpbs.org
690 690
691 691 .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/ssh-agent
General Comments 0
You need to be logged in to leave comments. Login now