##// END OF EJS Templates
rename core.newapplication -> core.application
MinRK -
Show More
This diff has been collapsed as it changes many lines, (719 lines changed) Show them Hide them
@@ -1,468 +1,429 b''
1 1 # encoding: utf-8
2 2 """
3 3 An application for IPython.
4 4
5 5 All top-level applications should use the classes in this module for
6 6 handling configuration and creating componenets.
7 7
8 8 The job of an :class:`Application` is to create the master configuration
9 9 object and then create the configurable objects, passing the config to them.
10 10
11 11 Authors:
12 12
13 13 * Brian Granger
14 14 * Fernando Perez
15 * Min RK
15 16
16 17 Notes
17 18 -----
18 19 """
19 20
20 21 #-----------------------------------------------------------------------------
21 22 # Copyright (C) 2008-2009 The IPython Development Team
22 23 #
23 24 # Distributed under the terms of the BSD License. The full license is in
24 25 # the file COPYING, distributed as part of this software.
25 26 #-----------------------------------------------------------------------------
26 27
27 28 #-----------------------------------------------------------------------------
28 29 # Imports
29 30 #-----------------------------------------------------------------------------
30 31
31 32 import logging
32 33 import os
34 import shutil
33 35 import sys
34 36
37 from IPython.config.application import Application
38 from IPython.config.configurable import Configurable
39 from IPython.config.loader import Config
35 40 from IPython.core import release, crashhandler
36 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
37 from IPython.config.loader import (
38 PyFileConfigLoader,
39 ArgParseConfigLoader,
40 Config,
41 )
41 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir, expand_path
42 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict
42 43
43 44 #-----------------------------------------------------------------------------
44 45 # Classes and functions
45 46 #-----------------------------------------------------------------------------
46 47
47 class ApplicationError(Exception):
48
49 #-----------------------------------------------------------------------------
50 # Module errors
51 #-----------------------------------------------------------------------------
52
53 class ProfileDirError(Exception):
48 54 pass
49 55
50 56
51 class BaseAppConfigLoader(ArgParseConfigLoader):
52 """Default command line options for IPython based applications."""
53
54 def _add_ipython_dir(self, parser):
55 """Add the --ipython-dir option to the parser."""
56 paa = parser.add_argument
57 paa('--ipython-dir',
58 dest='Global.ipython_dir',type=unicode,
59 help=
60 """Set to override default location of the IPython directory
61 IPYTHON_DIR, stored as Global.ipython_dir. This can also be
62 specified through the environment variable IPYTHON_DIR.""",
63 metavar='Global.ipython_dir')
64
65 def _add_log_level(self, parser):
66 """Add the --log-level option to the parser."""
67 paa = parser.add_argument
68 paa('--log-level',
69 dest="Global.log_level",type=int,
70 help='Set the log level (0,10,20,30,40,50). Default is 30.',
71 metavar='Global.log_level')
72
73 def _add_version(self, parser):
74 """Add the --version option to the parser."""
75 parser.add_argument('--version', action="version",
76 version=self.version)
77
78 def _add_arguments(self):
79 self._add_ipython_dir(self.parser)
80 self._add_log_level(self.parser)
81 try: # Old versions of argparse don't have a version action
82 self._add_version(self.parser)
83 except Exception:
84 pass
85
86
87 class Application(object):
88 """Load a config, construct configurables and set them running.
89
90 The configuration of an application can be done via three different Config
91 objects, which are loaded and ultimately merged into a single one used
92 from that point on by the app. These are:
93
94 1. default_config: internal defaults, implemented in code.
95 2. file_config: read from the filesystem.
96 3. command_line_config: read from the system's command line flags.
97
98 During initialization, 3 is actually read before 2, since at the
99 command-line one may override the location of the file to be read. But the
100 above is the order in which the merge is made.
101 """
57 #-----------------------------------------------------------------------------
58 # Class for managing profile directories
59 #-----------------------------------------------------------------------------
102 60
103 name = u'ipython'
104 description = 'IPython: an enhanced interactive Python shell.'
105 #: Usage message printed by argparse. If None, auto-generate
106 usage = None
107 #: The command line config loader. Subclass of ArgParseConfigLoader.
108 command_line_loader = BaseAppConfigLoader
109 #: The name of the config file to load, determined at runtime
110 config_file_name = None
111 #: The name of the default config file. Track separately from the actual
112 #: name because some logic happens only if we aren't using the default.
113 default_config_file_name = u'ipython_config.py'
114 default_log_level = logging.WARN
115 #: Set by --profile option
116 profile_name = None
117 #: User's ipython directory, typically ~/.ipython or ~/.config/ipython/
118 ipython_dir = None
119 #: Internal defaults, implemented in code.
120 default_config = None
121 #: Read from the filesystem.
122 file_config = None
123 #: Read from the system's command line flags.
124 command_line_config = None
125 #: The final config that will be passed to the main object.
126 master_config = None
127 #: A reference to the argv to be used (typically ends up being sys.argv[1:])
128 argv = None
129 #: extra arguments computed by the command-line loader
130 extra_args = None
131 #: The class to use as the crash handler.
132 crash_handler_class = crashhandler.CrashHandler
133
134 # Private attributes
135 _exiting = False
136 _initialized = False
137
138 def __init__(self, argv=None):
139 self.argv = sys.argv[1:] if argv is None else argv
140 self.init_logger()
141
142 def init_logger(self):
143 self.log = logging.getLogger(self.__class__.__name__)
144 # This is used as the default until the command line arguments are read.
145 self.log.setLevel(self.default_log_level)
146 self._log_handler = logging.StreamHandler()
147 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
148 self._log_handler.setFormatter(self._log_formatter)
149 self.log.addHandler(self._log_handler)
150
151 def _set_log_level(self, level):
152 self.log.setLevel(level)
153
154 def _get_log_level(self):
155 return self.log.level
156
157 log_level = property(_get_log_level, _set_log_level)
158
159 def initialize(self):
160 """Initialize the application.
161
162 Loads all configuration information and sets all application state, but
163 does not start any relevant processing (typically some kind of event
164 loop).
165
166 Once this method has been called, the application is flagged as
167 initialized and the method becomes a no-op."""
168
169 if self._initialized:
170 return
61 class ProfileDir(Configurable):
62 """An object to manage the profile directory and its resources.
171 63
172 # The first part is protected with an 'attempt' wrapper, that will log
173 # failures with the basic system traceback machinery. Once our crash
174 # handler is in place, we can let any subsequent exception propagate,
175 # as our handler will log it with much better detail than the default.
176 self.attempt(self.create_crash_handler)
177
178 # Configuration phase
179 # Default config (internally hardwired in application code)
180 self.create_default_config()
181 self.log_default_config()
182 self.set_default_config_log_level()
183
184 # Command-line config
185 self.pre_load_command_line_config()
186 self.load_command_line_config()
187 self.set_command_line_config_log_level()
188 self.post_load_command_line_config()
189 self.log_command_line_config()
190
191 # Find resources needed for filesystem access, using information from
192 # the above two
193 self.find_ipython_dir()
194 self.find_resources()
195 self.find_config_file_name()
196 self.find_config_file_paths()
197
198 # File-based config
199 self.pre_load_file_config()
200 self.load_file_config()
201 self.set_file_config_log_level()
202 self.post_load_file_config()
203 self.log_file_config()
204
205 # Merge all config objects into a single one the app can then use
206 self.merge_configs()
207 self.log_master_config()
208
209 # Construction phase
210 self.pre_construct()
211 self.construct()
212 self.post_construct()
213
214 # Done, flag as such and
215 self._initialized = True
216
217 def start(self):
218 """Start the application."""
219 self.initialize()
220 self.start_app()
64 The profile directory is used by all IPython applications, to manage
65 configuration, logging and security.
221 66
222 #-------------------------------------------------------------------------
223 # Various stages of Application creation
224 #-------------------------------------------------------------------------
67 This object knows how to find, create and manage these directories. This
68 should be used by any code that wants to handle profiles.
69 """
225 70
226 def create_crash_handler(self):
227 """Create a crash handler, typically setting sys.excepthook to it."""
228 self.crash_handler = self.crash_handler_class(self)
229 sys.excepthook = self.crash_handler
71 security_dir_name = Unicode('security')
72 log_dir_name = Unicode('log')
73 pid_dir_name = Unicode('pid')
74 security_dir = Unicode(u'')
75 log_dir = Unicode(u'')
76 pid_dir = Unicode(u'')
77
78 location = Unicode(u'', config=True,
79 help="""Set the profile location directly. This overrides the logic used by the
80 `profile` option.""",
81 )
230 82
231 def create_default_config(self):
232 """Create defaults that can't be set elsewhere.
83 _location_isset = Bool(False) # flag for detecting multiply set location
233 84
234 For the most part, we try to set default in the class attributes
235 of Configurables. But, defaults the top-level Application (which is
236 not a HasTraits or Configurables) are not set in this way. Instead
237 we set them here. The Global section is for variables like this that
238 don't belong to a particular configurable.
239 """
240 c = Config()
241 c.Global.ipython_dir = get_ipython_dir()
242 c.Global.log_level = self.log_level
243 self.default_config = c
85 def _location_changed(self, name, old, new):
86 if self._location_isset:
87 raise RuntimeError("Cannot set profile location more than once.")
88 self._location_isset = True
89 if not os.path.isdir(new):
90 os.makedirs(new)
91
92 # ensure config files exist:
93 self.security_dir = os.path.join(new, self.security_dir_name)
94 self.log_dir = os.path.join(new, self.log_dir_name)
95 self.pid_dir = os.path.join(new, self.pid_dir_name)
96 self.check_dirs()
244 97
245 def log_default_config(self):
246 self.log.debug('Default config loaded:')
247 self.log.debug(repr(self.default_config))
98 def _log_dir_changed(self, name, old, new):
99 self.check_log_dir()
248 100
249 def set_default_config_log_level(self):
250 try:
251 self.log_level = self.default_config.Global.log_level
252 except AttributeError:
253 # Fallback to the default_log_level class attribute
254 pass
255
256 def create_command_line_config(self):
257 """Create and return a command line config loader."""
258 return self.command_line_loader(
259 self.argv,
260 description=self.description,
261 version=release.version,
262 usage=self.usage
263 )
101 def check_log_dir(self):
102 if not os.path.isdir(self.log_dir):
103 os.mkdir(self.log_dir)
264 104
265 def pre_load_command_line_config(self):
266 """Do actions just before loading the command line config."""
267 pass
105 def _security_dir_changed(self, name, old, new):
106 self.check_security_dir()
268 107
269 def load_command_line_config(self):
270 """Load the command line config."""
271 loader = self.create_command_line_config()
272 self.command_line_config = loader.load_config()
273 self.extra_args = loader.get_extra_args()
108 def check_security_dir(self):
109 if not os.path.isdir(self.security_dir):
110 os.mkdir(self.security_dir, 0700)
111 else:
112 os.chmod(self.security_dir, 0700)
274 113
275 def set_command_line_config_log_level(self):
276 try:
277 self.log_level = self.command_line_config.Global.log_level
278 except AttributeError:
279 pass
114 def _pid_dir_changed(self, name, old, new):
115 self.check_pid_dir()
280 116
281 def post_load_command_line_config(self):
282 """Do actions just after loading the command line config."""
283 pass
117 def check_pid_dir(self):
118 if not os.path.isdir(self.pid_dir):
119 os.mkdir(self.pid_dir, 0700)
120 else:
121 os.chmod(self.pid_dir, 0700)
284 122
285 def log_command_line_config(self):
286 self.log.debug("Command line config loaded:")
287 self.log.debug(repr(self.command_line_config))
123 def check_dirs(self):
124 self.check_security_dir()
125 self.check_log_dir()
126 self.check_pid_dir()
288 127
289 def find_ipython_dir(self):
290 """Set the IPython directory.
128 def copy_config_file(self, config_file, path=None, overwrite=False):
129 """Copy a default config file into the active profile directory.
291 130
292 This sets ``self.ipython_dir``, but the actual value that is passed to
293 the application is kept in either ``self.default_config`` or
294 ``self.command_line_config``. This also adds ``self.ipython_dir`` to
295 ``sys.path`` so config files there can be referenced by other config
296 files.
131 Default configuration files are kept in :mod:`IPython.config.default`.
132 This function moves these from that location to the working profile
133 directory.
297 134 """
298
299 try:
300 self.ipython_dir = self.command_line_config.Global.ipython_dir
301 except AttributeError:
302 self.ipython_dir = self.default_config.Global.ipython_dir
303 sys.path.append(os.path.abspath(self.ipython_dir))
304 if not os.path.isdir(self.ipython_dir):
305 os.makedirs(self.ipython_dir, mode=0777)
306 self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir)
307
308 def find_resources(self):
309 """Find other resources that need to be in place.
310
311 Things like cluster directories need to be in place to find the
312 config file. These happen right after the IPython directory has
313 been set.
135 dst = os.path.join(self.location, config_file)
136 if os.path.isfile(dst) and not overwrite:
137 return
138 if path is None:
139 path = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
140 src = os.path.join(path, config_file)
141 shutil.copy(src, dst)
142
143 @classmethod
144 def create_profile_dir(cls, profile_dir, config=None):
145 """Create a new profile directory given a full path.
146
147 Parameters
148 ----------
149 profile_dir : str
150 The full path to the profile directory. If it does exist, it will
151 be used. If not, it will be created.
152 """
153 return cls(location=profile_dir, config=config)
154
155 @classmethod
156 def create_profile_dir_by_name(cls, path, name=u'default', config=None):
157 """Create a profile dir by profile name and path.
158
159 Parameters
160 ----------
161 path : unicode
162 The path (directory) to put the profile directory in.
163 name : unicode
164 The name of the profile. The name of the profile directory will
165 be "profile_<profile>".
314 166 """
315 pass
167 if not os.path.isdir(path):
168 raise ProfileDirError('Directory not found: %s' % path)
169 profile_dir = os.path.join(path, u'profile_' + name)
170 return cls(location=profile_dir, config=config)
171
172 @classmethod
173 def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None):
174 """Find an existing profile dir by profile name, return its ProfileDir.
175
176 This searches through a sequence of paths for a profile dir. If it
177 is not found, a :class:`ProfileDirError` exception will be raised.
178
179 The search path algorithm is:
180 1. ``os.getcwd()``
181 2. ``ipython_dir``
182
183 Parameters
184 ----------
185 ipython_dir : unicode or str
186 The IPython directory to use.
187 name : unicode or str
188 The name of the profile. The name of the profile directory
189 will be "profile_<profile>".
190 """
191 dirname = u'profile_' + name
192 paths = [os.getcwdu(), ipython_dir]
193 for p in paths:
194 profile_dir = os.path.join(p, dirname)
195 if os.path.isdir(profile_dir):
196 return cls(location=profile_dir, config=config)
197 else:
198 raise ProfileDirError('Profile directory not found in paths: %s' % dirname)
316 199
317 def find_config_file_name(self):
318 """Find the config file name for this application.
200 @classmethod
201 def find_profile_dir(cls, profile_dir, config=None):
202 """Find/create a profile dir and return its ProfileDir.
319 203
320 This must set ``self.config_file_name`` to the filename of the
321 config file to use (just the filename). The search paths for the
322 config file are set in :meth:`find_config_file_paths` and then passed
323 to the config file loader where they are resolved to an absolute path.
204 This will create the profile directory if it doesn't exist.
324 205
325 If a profile has been set at the command line, this will resolve it.
206 Parameters
207 ----------
208 profile_dir : unicode or str
209 The path of the profile directory. This is expanded using
210 :func:`IPython.utils.genutils.expand_path`.
326 211 """
327 try:
328 self.config_file_name = self.command_line_config.Global.config_file
329 except AttributeError:
330 pass
331 else:
332 return
212 profile_dir = expand_path(profile_dir)
213 if not os.path.isdir(profile_dir):
214 raise ProfileDirError('Profile directory not found: %s' % profile_dir)
215 return cls(location=profile_dir, config=config)
333 216
334 try:
335 self.profile_name = self.command_line_config.Global.profile
336 except AttributeError:
337 # Just use the default as there is no profile
338 self.config_file_name = self.default_config_file_name
339 else:
340 # Use the default config file name and profile name if set
341 # to determine the used config file name.
342 name_parts = self.default_config_file_name.split('.')
343 name_parts.insert(1, u'_' + self.profile_name + u'.')
344 self.config_file_name = ''.join(name_parts)
345 217
346 def find_config_file_paths(self):
347 """Set the search paths for resolving the config file.
218 #-----------------------------------------------------------------------------
219 # Base Application Class
220 #-----------------------------------------------------------------------------
221
222 # aliases and flags
223
224 base_aliases = dict(
225 profile='BaseIPythonApplication.profile',
226 ipython_dir='BaseIPythonApplication.ipython_dir',
227 )
348 228
349 This must set ``self.config_file_paths`` to a sequence of search
350 paths to pass to the config file loader.
229 base_flags = dict(
230 debug = ({'Application' : {'log_level' : logging.DEBUG}},
231 "set log level to logging.DEBUG (maximize logging output)"),
232 quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
233 "set log level to logging.CRITICAL (minimize logging output)"),
234 init = ({'BaseIPythonApplication' : {
235 'copy_config_files' : True,
236 'auto_create' : True}
237 }, "Initialize profile with default config files")
238 )
239
240
241 class BaseIPythonApplication(Application):
242
243 name = Unicode(u'ipython')
244 description = Unicode(u'IPython: an enhanced interactive Python shell.')
245 version = Unicode(release.version)
246
247 aliases = Dict(base_aliases)
248 flags = Dict(base_flags)
249
250 # Track whether the config_file has changed,
251 # because some logic happens only if we aren't using the default.
252 config_file_specified = Bool(False)
253
254 config_file_name = Unicode(u'ipython_config.py')
255 def _config_file_name_changed(self, name, old, new):
256 if new != old:
257 self.config_file_specified = True
258
259 # The directory that contains IPython's builtin profiles.
260 builtin_profile_dir = Unicode(
261 os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
262 )
263
264 config_file_paths = List(Unicode)
265 def _config_file_paths_default(self):
266 return [os.getcwdu()]
267
268 profile = Unicode(u'default', config=True,
269 help="""The IPython profile to use."""
270 )
271 def _profile_changed(self, name, old, new):
272 self.builtin_profile_dir = os.path.join(
273 get_ipython_package_dir(), u'config', u'profile', new
274 )
275
276
277 ipython_dir = Unicode(get_ipython_dir(), config=True,
278 help="""
279 The name of the IPython directory. This directory is used for logging
280 configuration (through profiles), history storage, etc. The default
281 is usually $HOME/.ipython. This options can also be specified through
282 the environment variable IPYTHON_DIR.
351 283 """
352 # Include our own profiles directory last, so that users can still find
353 # our shipped copies of builtin profiles even if they don't have them
354 # in their local ipython directory.
355 prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
356 self.config_file_paths = (os.getcwdu(), self.ipython_dir, prof_dir)
284 )
285
286 overwrite = Bool(False, config=True,
287 help="""Whether to overwrite existing config files when copying""")
288 auto_create = Bool(False, config=True,
289 help="""Whether to create profile dir if it doesn't exist""")
290
291 config_files = List(Unicode)
292 def _config_files_default(self):
293 return [u'ipython_config.py']
294
295 copy_config_files = Bool(False, config=True,
296 help="""Whether to copy the default config files into the profile dir.""")
297
298 # The class to use as the crash handler.
299 crash_handler_class = Type(crashhandler.CrashHandler)
300
301 def __init__(self, **kwargs):
302 super(BaseIPythonApplication, self).__init__(**kwargs)
303 # ensure even default IPYTHON_DIR exists
304 if not os.path.exists(self.ipython_dir):
305 self._ipython_dir_changed('ipython_dir', self.ipython_dir, self.ipython_dir)
306
307 #-------------------------------------------------------------------------
308 # Various stages of Application creation
309 #-------------------------------------------------------------------------
357 310
358 def pre_load_file_config(self):
359 """Do actions before the config file is loaded."""
360 pass
311 def init_crash_handler(self):
312 """Create a crash handler, typically setting sys.excepthook to it."""
313 self.crash_handler = self.crash_handler_class(self)
314 sys.excepthook = self.crash_handler
361 315
362 def load_file_config(self, suppress_errors=True):
316 def _ipython_dir_changed(self, name, old, new):
317 if old in sys.path:
318 sys.path.remove(old)
319 sys.path.append(os.path.abspath(new))
320 if not os.path.isdir(new):
321 os.makedirs(new, mode=0777)
322 readme = os.path.join(new, 'README')
323 if not os.path.exists(readme):
324 path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
325 shutil.copy(os.path.join(path, 'README'), readme)
326 self.log.debug("IPYTHON_DIR set to: %s" % new)
327
328 def load_config_file(self, suppress_errors=True):
363 329 """Load the config file.
364
365 This tries to load the config file from disk. If successful, the
366 ``CONFIG_FILE`` config variable is set to the resolved config file
367 location. If not successful, an empty config is used.
368
330
369 331 By default, errors in loading config are handled, and a warning
370 332 printed on screen. For testing, the suppress_errors option is set
371 333 to False, so errors will make tests fail.
372 334 """
373 335 self.log.debug("Attempting to load config file: %s" %
374 336 self.config_file_name)
375 loader = PyFileConfigLoader(self.config_file_name,
376 path=self.config_file_paths)
377 337 try:
378 self.file_config = loader.load_config()
379 self.file_config.Global.config_file = loader.full_filename
338 Application.load_config_file(
339 self,
340 self.config_file_name,
341 path=self.config_file_paths
342 )
380 343 except IOError:
381 344 # Only warn if the default config file was NOT being used.
382 if not self.config_file_name==self.default_config_file_name:
345 if self.config_file_specified:
383 346 self.log.warn("Config file not found, skipping: %s" %
384 self.config_file_name, exc_info=True)
385 self.file_config = Config()
347 self.config_file_name)
386 348 except:
387 if not suppress_errors: # For testing purposes
349 # For testing purposes.
350 if not suppress_errors:
388 351 raise
389 352 self.log.warn("Error loading config file: %s" %
390 353 self.config_file_name, exc_info=True)
391 self.file_config = Config()
392 354
393 def set_file_config_log_level(self):
394 # We need to keeep self.log_level updated. But we only use the value
395 # of the file_config if a value was not specified at the command
396 # line, because the command line overrides everything.
397 if not hasattr(self.command_line_config.Global, 'log_level'):
355 def init_profile_dir(self):
356 """initialize the profile dir"""
357 try:
358 # location explicitly specified:
359 location = self.config.ProfileDir.location
360 except AttributeError:
361 # location not specified, find by profile name
398 362 try:
399 self.log_level = self.file_config.Global.log_level
400 except AttributeError:
401 pass # Use existing value
402
403 def post_load_file_config(self):
404 """Do actions after the config file is loaded."""
405 pass
406
407 def log_file_config(self):
408 if hasattr(self.file_config.Global, 'config_file'):
409 self.log.debug("Config file loaded: %s" %
410 self.file_config.Global.config_file)
411 self.log.debug(repr(self.file_config))
412
413 def merge_configs(self):
414 """Merge the default, command line and file config objects."""
415 config = Config()
416 config._merge(self.default_config)
417 config._merge(self.file_config)
418 config._merge(self.command_line_config)
419
420 # XXX fperez - propose to Brian we rename master_config to simply
421 # config, I think this is going to be heavily used in examples and
422 # application code and the name is shorter/easier to find/remember.
423 # For now, just alias it...
424 self.master_config = config
425 self.config = config
426
427 def log_master_config(self):
428 self.log.debug("Master config created:")
429 self.log.debug(repr(self.master_config))
430
431 def pre_construct(self):
432 """Do actions after the config has been built, but before construct."""
433 pass
434
435 def construct(self):
436 """Construct the main objects that make up this app."""
437 self.log.debug("Constructing main objects for application")
438
439 def post_construct(self):
440 """Do actions after construct, but before starting the app."""
441 pass
442
443 def start_app(self):
444 """Actually start the app."""
445 self.log.debug("Starting application")
446
447 #-------------------------------------------------------------------------
448 # Utility methods
449 #-------------------------------------------------------------------------
450
451 def exit(self, exit_status=0):
452 if self._exiting:
453 pass
363 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
364 except ProfileDirError:
365 # not found, maybe create it (always create default profile)
366 if self.auto_create or self.profile=='default':
367 try:
368 p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
369 except ProfileDirError:
370 self.log.fatal("Could not create profile: %r"%self.profile)
371 self.exit(1)
372 else:
373 self.log.info("Created profile dir: %r"%p.location)
374 else:
375 self.log.fatal("Profile %r not found."%self.profile)
376 self.exit(1)
377 else:
378 self.log.info("Using existing profile dir: %r"%p.location)
454 379 else:
455 self.log.debug("Exiting application: %s" % self.name)
456 self._exiting = True
457 sys.exit(exit_status)
458
459 def attempt(self, func):
460 try:
461 func()
462 except SystemExit:
463 raise
464 except:
465 self.log.critical("Aborting application: %s" % self.name,
466 exc_info=True)
467 self.exit(0)
380 # location is fully specified
381 try:
382 p = ProfileDir.find_profile_dir(location, self.config)
383 except ProfileDirError:
384 # not found, maybe create it
385 if self.auto_create:
386 try:
387 p = ProfileDir.create_profile_dir(location, self.config)
388 except ProfileDirError:
389 self.log.fatal("Could not create profile directory: %r"%location)
390 self.exit(1)
391 else:
392 self.log.info("Creating new profile dir: %r"%location)
393 else:
394 self.log.fatal("Profile directory %r not found."%location)
395 self.exit(1)
396 else:
397 self.log.info("Using existing profile dir: %r"%location)
398
399 self.profile_dir = p
400 self.config_file_paths.append(p.location)
401
402 def init_config_files(self):
403 """[optionally] copy default config files into profile dir."""
404 # copy config files
405 if self.copy_config_files:
406 path = self.builtin_profile_dir
407 src = self.profile
408 if not os.path.exists(path):
409 # use default if new profile doesn't have a preset
410 path = None
411 src = 'default'
412
413 self.log.debug("Staging %s config files into %r [overwrite=%s]"%(
414 src, self.profile_dir.location, self.overwrite)
415 )
416
417 for cfg in self.config_files:
418 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
419
420 def initialize(self, argv=None):
421 self.init_crash_handler()
422 self.parse_command_line(argv)
423 cl_config = self.config
424 self.init_profile_dir()
425 self.init_config_files()
426 self.load_config_file()
427 # enforce cl-opts override configfile opts:
428 self.update_config(cl_config)
468 429
@@ -1,2553 +1,2553 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Main IPython class."""
3 3
4 4 #-----------------------------------------------------------------------------
5 5 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
6 6 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
7 7 # Copyright (C) 2008-2011 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Imports
15 15 #-----------------------------------------------------------------------------
16 16
17 17 from __future__ import with_statement
18 18 from __future__ import absolute_import
19 19
20 20 import __builtin__
21 21 import __future__
22 22 import abc
23 23 import ast
24 24 import atexit
25 25 import codeop
26 26 import inspect
27 27 import os
28 28 import re
29 29 import sys
30 30 import tempfile
31 31 import types
32 32 from contextlib import nested
33 33
34 34 from IPython.config.configurable import SingletonConfigurable
35 35 from IPython.core import debugger, oinspect
36 36 from IPython.core import history as ipcorehist
37 37 from IPython.core import page
38 38 from IPython.core import prefilter
39 39 from IPython.core import shadowns
40 40 from IPython.core import ultratb
41 41 from IPython.core.alias import AliasManager, AliasError
42 42 from IPython.core.autocall import ExitAutocall
43 43 from IPython.core.builtin_trap import BuiltinTrap
44 44 from IPython.core.compilerop import CachingCompiler
45 45 from IPython.core.display_trap import DisplayTrap
46 46 from IPython.core.displayhook import DisplayHook
47 47 from IPython.core.displaypub import DisplayPublisher
48 48 from IPython.core.error import TryNext, UsageError
49 49 from IPython.core.extensions import ExtensionManager
50 50 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
51 51 from IPython.core.formatters import DisplayFormatter
52 52 from IPython.core.history import HistoryManager
53 53 from IPython.core.inputsplitter import IPythonInputSplitter
54 54 from IPython.core.logger import Logger
55 55 from IPython.core.macro import Macro
56 56 from IPython.core.magic import Magic
57 from IPython.core.newapplication import ProfileDir
57 from IPython.core.application import ProfileDir
58 58 from IPython.core.payload import PayloadManager
59 59 from IPython.core.plugin import PluginManager
60 60 from IPython.core.prefilter import PrefilterManager, ESC_MAGIC
61 61 from IPython.external.Itpl import ItplNS
62 62 from IPython.utils import PyColorize
63 63 from IPython.utils import io
64 64 from IPython.utils.doctestreload import doctest_reload
65 65 from IPython.utils.io import ask_yes_no, rprint
66 66 from IPython.utils.ipstruct import Struct
67 67 from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError
68 68 from IPython.utils.pickleshare import PickleShareDB
69 69 from IPython.utils.process import system, getoutput
70 70 from IPython.utils.strdispatch import StrDispatch
71 71 from IPython.utils.syspathcontext import prepended_to_syspath
72 72 from IPython.utils.text import num_ini_spaces, format_screen, LSString, SList
73 73 from IPython.utils.traitlets import (Int, Str, CBool, CaselessStrEnum, Enum,
74 74 List, Unicode, Instance, Type)
75 75 from IPython.utils.warn import warn, error, fatal
76 76 import IPython.core.hooks
77 77
78 78 #-----------------------------------------------------------------------------
79 79 # Globals
80 80 #-----------------------------------------------------------------------------
81 81
82 82 # compiled regexps for autoindent management
83 83 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
84 84
85 85 #-----------------------------------------------------------------------------
86 86 # Utilities
87 87 #-----------------------------------------------------------------------------
88 88
89 89 # store the builtin raw_input globally, and use this always, in case user code
90 90 # overwrites it (like wx.py.PyShell does)
91 91 raw_input_original = raw_input
92 92
93 93 def softspace(file, newvalue):
94 94 """Copied from code.py, to remove the dependency"""
95 95
96 96 oldvalue = 0
97 97 try:
98 98 oldvalue = file.softspace
99 99 except AttributeError:
100 100 pass
101 101 try:
102 102 file.softspace = newvalue
103 103 except (AttributeError, TypeError):
104 104 # "attribute-less object" or "read-only attributes"
105 105 pass
106 106 return oldvalue
107 107
108 108
109 109 def no_op(*a, **kw): pass
110 110
111 111 class SpaceInInput(Exception): pass
112 112
113 113 class Bunch: pass
114 114
115 115
116 116 def get_default_colors():
117 117 if sys.platform=='darwin':
118 118 return "LightBG"
119 119 elif os.name=='nt':
120 120 return 'Linux'
121 121 else:
122 122 return 'Linux'
123 123
124 124
125 125 class SeparateStr(Str):
126 126 """A Str subclass to validate separate_in, separate_out, etc.
127 127
128 128 This is a Str based trait that converts '0'->'' and '\\n'->'\n'.
129 129 """
130 130
131 131 def validate(self, obj, value):
132 132 if value == '0': value = ''
133 133 value = value.replace('\\n','\n')
134 134 return super(SeparateStr, self).validate(obj, value)
135 135
136 136
137 137 class ReadlineNoRecord(object):
138 138 """Context manager to execute some code, then reload readline history
139 139 so that interactive input to the code doesn't appear when pressing up."""
140 140 def __init__(self, shell):
141 141 self.shell = shell
142 142 self._nested_level = 0
143 143
144 144 def __enter__(self):
145 145 if self._nested_level == 0:
146 146 try:
147 147 self.orig_length = self.current_length()
148 148 self.readline_tail = self.get_readline_tail()
149 149 except (AttributeError, IndexError): # Can fail with pyreadline
150 150 self.orig_length, self.readline_tail = 999999, []
151 151 self._nested_level += 1
152 152
153 153 def __exit__(self, type, value, traceback):
154 154 self._nested_level -= 1
155 155 if self._nested_level == 0:
156 156 # Try clipping the end if it's got longer
157 157 try:
158 158 e = self.current_length() - self.orig_length
159 159 if e > 0:
160 160 for _ in range(e):
161 161 self.shell.readline.remove_history_item(self.orig_length)
162 162
163 163 # If it still doesn't match, just reload readline history.
164 164 if self.current_length() != self.orig_length \
165 165 or self.get_readline_tail() != self.readline_tail:
166 166 self.shell.refill_readline_hist()
167 167 except (AttributeError, IndexError):
168 168 pass
169 169 # Returning False will cause exceptions to propagate
170 170 return False
171 171
172 172 def current_length(self):
173 173 return self.shell.readline.get_current_history_length()
174 174
175 175 def get_readline_tail(self, n=10):
176 176 """Get the last n items in readline history."""
177 177 end = self.shell.readline.get_current_history_length() + 1
178 178 start = max(end-n, 1)
179 179 ghi = self.shell.readline.get_history_item
180 180 return [ghi(x) for x in range(start, end)]
181 181
182 182
183 183 _autocall_help = """
184 184 Make IPython automatically call any callable object even if
185 185 you didn't type explicit parentheses. For example, 'str 43' becomes 'str(43)'
186 186 automatically. The value can be '0' to disable the feature, '1' for 'smart'
187 187 autocall, where it is not applied if there are no more arguments on the line,
188 188 and '2' for 'full' autocall, where all callable objects are automatically
189 189 called (even if no arguments are present). The default is '1'.
190 190 """
191 191
192 192 #-----------------------------------------------------------------------------
193 193 # Main IPython class
194 194 #-----------------------------------------------------------------------------
195 195
196 196 class InteractiveShell(SingletonConfigurable, Magic):
197 197 """An enhanced, interactive shell for Python."""
198 198
199 199 _instance = None
200 200
201 201 autocall = Enum((0,1,2), default_value=1, config=True, help=
202 202 """
203 203 Make IPython automatically call any callable object even if you didn't
204 204 type explicit parentheses. For example, 'str 43' becomes 'str(43)'
205 205 automatically. The value can be '0' to disable the feature, '1' for
206 206 'smart' autocall, where it is not applied if there are no more
207 207 arguments on the line, and '2' for 'full' autocall, where all callable
208 208 objects are automatically called (even if no arguments are present).
209 209 The default is '1'.
210 210 """
211 211 )
212 212 # TODO: remove all autoindent logic and put into frontends.
213 213 # We can't do this yet because even runlines uses the autoindent.
214 214 autoindent = CBool(True, config=True, help=
215 215 """
216 216 Autoindent IPython code entered interactively.
217 217 """
218 218 )
219 219 automagic = CBool(True, config=True, help=
220 220 """
221 221 Enable magic commands to be called without the leading %.
222 222 """
223 223 )
224 224 cache_size = Int(1000, config=True, help=
225 225 """
226 226 Set the size of the output cache. The default is 1000, you can
227 227 change it permanently in your config file. Setting it to 0 completely
228 228 disables the caching system, and the minimum value accepted is 20 (if
229 229 you provide a value less than 20, it is reset to 0 and a warning is
230 230 issued). This limit is defined because otherwise you'll spend more
231 231 time re-flushing a too small cache than working
232 232 """
233 233 )
234 234 color_info = CBool(True, config=True, help=
235 235 """
236 236 Use colors for displaying information about objects. Because this
237 237 information is passed through a pager (like 'less'), and some pagers
238 238 get confused with color codes, this capability can be turned off.
239 239 """
240 240 )
241 241 colors = CaselessStrEnum(('NoColor','LightBG','Linux'),
242 242 default_value=get_default_colors(), config=True,
243 243 help="Set the color scheme (NoColor, Linux, or LightBG)."
244 244 )
245 245 debug = CBool(False, config=True)
246 246 deep_reload = CBool(False, config=True, help=
247 247 """
248 248 Enable deep (recursive) reloading by default. IPython can use the
249 249 deep_reload module which reloads changes in modules recursively (it
250 250 replaces the reload() function, so you don't need to change anything to
251 251 use it). deep_reload() forces a full reload of modules whose code may
252 252 have changed, which the default reload() function does not. When
253 253 deep_reload is off, IPython will use the normal reload(), but
254 254 deep_reload will still be available as dreload().
255 255 """
256 256 )
257 257 display_formatter = Instance(DisplayFormatter)
258 258 displayhook_class = Type(DisplayHook)
259 259 display_pub_class = Type(DisplayPublisher)
260 260
261 261 exit_now = CBool(False)
262 262 exiter = Instance(ExitAutocall)
263 263 def _exiter_default(self):
264 264 return ExitAutocall(self)
265 265 # Monotonically increasing execution counter
266 266 execution_count = Int(1)
267 267 filename = Unicode("<ipython console>")
268 268 ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
269 269
270 270 # Input splitter, to split entire cells of input into either individual
271 271 # interactive statements or whole blocks.
272 272 input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
273 273 (), {})
274 274 logstart = CBool(False, config=True, help=
275 275 """
276 276 Start logging to the default log file.
277 277 """
278 278 )
279 279 logfile = Unicode('', config=True, help=
280 280 """
281 281 The name of the logfile to use.
282 282 """
283 283 )
284 284 logappend = Unicode('', config=True, help=
285 285 """
286 286 Start logging to the given file in append mode.
287 287 """
288 288 )
289 289 object_info_string_level = Enum((0,1,2), default_value=0,
290 290 config=True)
291 291 pdb = CBool(False, config=True, help=
292 292 """
293 293 Automatically call the pdb debugger after every exception.
294 294 """
295 295 )
296 296
297 297 prompt_in1 = Str('In [\\#]: ', config=True)
298 298 prompt_in2 = Str(' .\\D.: ', config=True)
299 299 prompt_out = Str('Out[\\#]: ', config=True)
300 300 prompts_pad_left = CBool(True, config=True)
301 301 quiet = CBool(False, config=True)
302 302
303 303 history_length = Int(10000, config=True)
304 304
305 305 # The readline stuff will eventually be moved to the terminal subclass
306 306 # but for now, we can't do that as readline is welded in everywhere.
307 307 readline_use = CBool(True, config=True)
308 308 readline_merge_completions = CBool(True, config=True)
309 309 readline_omit__names = Enum((0,1,2), default_value=2, config=True)
310 310 readline_remove_delims = Str('-/~', config=True)
311 311 # don't use \M- bindings by default, because they
312 312 # conflict with 8-bit encodings. See gh-58,gh-88
313 313 readline_parse_and_bind = List([
314 314 'tab: complete',
315 315 '"\C-l": clear-screen',
316 316 'set show-all-if-ambiguous on',
317 317 '"\C-o": tab-insert',
318 318 '"\C-r": reverse-search-history',
319 319 '"\C-s": forward-search-history',
320 320 '"\C-p": history-search-backward',
321 321 '"\C-n": history-search-forward',
322 322 '"\e[A": history-search-backward',
323 323 '"\e[B": history-search-forward',
324 324 '"\C-k": kill-line',
325 325 '"\C-u": unix-line-discard',
326 326 ], allow_none=False, config=True)
327 327
328 328 # TODO: this part of prompt management should be moved to the frontends.
329 329 # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
330 330 separate_in = SeparateStr('\n', config=True)
331 331 separate_out = SeparateStr('', config=True)
332 332 separate_out2 = SeparateStr('', config=True)
333 333 wildcards_case_sensitive = CBool(True, config=True)
334 334 xmode = CaselessStrEnum(('Context','Plain', 'Verbose'),
335 335 default_value='Context', config=True)
336 336
337 337 # Subcomponents of InteractiveShell
338 338 alias_manager = Instance('IPython.core.alias.AliasManager')
339 339 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
340 340 builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap')
341 341 display_trap = Instance('IPython.core.display_trap.DisplayTrap')
342 342 extension_manager = Instance('IPython.core.extensions.ExtensionManager')
343 343 plugin_manager = Instance('IPython.core.plugin.PluginManager')
344 344 payload_manager = Instance('IPython.core.payload.PayloadManager')
345 345 history_manager = Instance('IPython.core.history.HistoryManager')
346 346
347 profile_dir = Instance('IPython.core.newapplication.ProfileDir')
347 profile_dir = Instance('IPython.core.application.ProfileDir')
348 348 @property
349 349 def profile(self):
350 350 if self.profile_dir is not None:
351 351 name = os.path.basename(self.profile_dir.location)
352 352 return name.replace('profile_','')
353 353
354 354
355 355 # Private interface
356 356 _post_execute = Instance(dict)
357 357
358 358 def __init__(self, config=None, ipython_dir=None, profile_dir=None,
359 359 user_ns=None, user_global_ns=None,
360 360 custom_exceptions=((), None)):
361 361
362 362 # This is where traits with a config_key argument are updated
363 363 # from the values on config.
364 364 super(InteractiveShell, self).__init__(config=config)
365 365
366 366 # These are relatively independent and stateless
367 367 self.init_ipython_dir(ipython_dir)
368 368 self.init_profile_dir(profile_dir)
369 369 self.init_instance_attrs()
370 370 self.init_environment()
371 371
372 372 # Create namespaces (user_ns, user_global_ns, etc.)
373 373 self.init_create_namespaces(user_ns, user_global_ns)
374 374 # This has to be done after init_create_namespaces because it uses
375 375 # something in self.user_ns, but before init_sys_modules, which
376 376 # is the first thing to modify sys.
377 377 # TODO: When we override sys.stdout and sys.stderr before this class
378 378 # is created, we are saving the overridden ones here. Not sure if this
379 379 # is what we want to do.
380 380 self.save_sys_module_state()
381 381 self.init_sys_modules()
382 382
383 383 # While we're trying to have each part of the code directly access what
384 384 # it needs without keeping redundant references to objects, we have too
385 385 # much legacy code that expects ip.db to exist.
386 386 self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
387 387
388 388 self.init_history()
389 389 self.init_encoding()
390 390 self.init_prefilter()
391 391
392 392 Magic.__init__(self, self)
393 393
394 394 self.init_syntax_highlighting()
395 395 self.init_hooks()
396 396 self.init_pushd_popd_magic()
397 397 # self.init_traceback_handlers use to be here, but we moved it below
398 398 # because it and init_io have to come after init_readline.
399 399 self.init_user_ns()
400 400 self.init_logger()
401 401 self.init_alias()
402 402 self.init_builtins()
403 403
404 404 # pre_config_initialization
405 405
406 406 # The next section should contain everything that was in ipmaker.
407 407 self.init_logstart()
408 408
409 409 # The following was in post_config_initialization
410 410 self.init_inspector()
411 411 # init_readline() must come before init_io(), because init_io uses
412 412 # readline related things.
413 413 self.init_readline()
414 414 # init_completer must come after init_readline, because it needs to
415 415 # know whether readline is present or not system-wide to configure the
416 416 # completers, since the completion machinery can now operate
417 417 # independently of readline (e.g. over the network)
418 418 self.init_completer()
419 419 # TODO: init_io() needs to happen before init_traceback handlers
420 420 # because the traceback handlers hardcode the stdout/stderr streams.
421 421 # This logic in in debugger.Pdb and should eventually be changed.
422 422 self.init_io()
423 423 self.init_traceback_handlers(custom_exceptions)
424 424 self.init_prompts()
425 425 self.init_display_formatter()
426 426 self.init_display_pub()
427 427 self.init_displayhook()
428 428 self.init_reload_doctest()
429 429 self.init_magics()
430 430 self.init_pdb()
431 431 self.init_extension_manager()
432 432 self.init_plugin_manager()
433 433 self.init_payload()
434 434 self.hooks.late_startup_hook()
435 435 atexit.register(self.atexit_operations)
436 436
437 437 def get_ipython(self):
438 438 """Return the currently running IPython instance."""
439 439 return self
440 440
441 441 #-------------------------------------------------------------------------
442 442 # Trait changed handlers
443 443 #-------------------------------------------------------------------------
444 444
445 445 def _ipython_dir_changed(self, name, new):
446 446 if not os.path.isdir(new):
447 447 os.makedirs(new, mode = 0777)
448 448
449 449 def set_autoindent(self,value=None):
450 450 """Set the autoindent flag, checking for readline support.
451 451
452 452 If called with no arguments, it acts as a toggle."""
453 453
454 454 if not self.has_readline:
455 455 if os.name == 'posix':
456 456 warn("The auto-indent feature requires the readline library")
457 457 self.autoindent = 0
458 458 return
459 459 if value is None:
460 460 self.autoindent = not self.autoindent
461 461 else:
462 462 self.autoindent = value
463 463
464 464 #-------------------------------------------------------------------------
465 465 # init_* methods called by __init__
466 466 #-------------------------------------------------------------------------
467 467
468 468 def init_ipython_dir(self, ipython_dir):
469 469 if ipython_dir is not None:
470 470 self.ipython_dir = ipython_dir
471 471 return
472 472
473 473 self.ipython_dir = get_ipython_dir()
474 474
475 475 def init_profile_dir(self, profile_dir):
476 476 if profile_dir is not None:
477 477 self.profile_dir = profile_dir
478 478 return
479 479 self.profile_dir =\
480 480 ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
481 481
482 482 def init_instance_attrs(self):
483 483 self.more = False
484 484
485 485 # command compiler
486 486 self.compile = CachingCompiler()
487 487
488 488 # Make an empty namespace, which extension writers can rely on both
489 489 # existing and NEVER being used by ipython itself. This gives them a
490 490 # convenient location for storing additional information and state
491 491 # their extensions may require, without fear of collisions with other
492 492 # ipython names that may develop later.
493 493 self.meta = Struct()
494 494
495 495 # Temporary files used for various purposes. Deleted at exit.
496 496 self.tempfiles = []
497 497
498 498 # Keep track of readline usage (later set by init_readline)
499 499 self.has_readline = False
500 500
501 501 # keep track of where we started running (mainly for crash post-mortem)
502 502 # This is not being used anywhere currently.
503 503 self.starting_dir = os.getcwd()
504 504
505 505 # Indentation management
506 506 self.indent_current_nsp = 0
507 507
508 508 # Dict to track post-execution functions that have been registered
509 509 self._post_execute = {}
510 510
511 511 def init_environment(self):
512 512 """Any changes we need to make to the user's environment."""
513 513 pass
514 514
515 515 def init_encoding(self):
516 516 # Get system encoding at startup time. Certain terminals (like Emacs
517 517 # under Win32 have it set to None, and we need to have a known valid
518 518 # encoding to use in the raw_input() method
519 519 try:
520 520 self.stdin_encoding = sys.stdin.encoding or 'ascii'
521 521 except AttributeError:
522 522 self.stdin_encoding = 'ascii'
523 523
524 524 def init_syntax_highlighting(self):
525 525 # Python source parser/formatter for syntax highlighting
526 526 pyformat = PyColorize.Parser().format
527 527 self.pycolorize = lambda src: pyformat(src,'str',self.colors)
528 528
529 529 def init_pushd_popd_magic(self):
530 530 # for pushd/popd management
531 531 try:
532 532 self.home_dir = get_home_dir()
533 533 except HomeDirError, msg:
534 534 fatal(msg)
535 535
536 536 self.dir_stack = []
537 537
538 538 def init_logger(self):
539 539 self.logger = Logger(self.home_dir, logfname='ipython_log.py',
540 540 logmode='rotate')
541 541
542 542 def init_logstart(self):
543 543 """Initialize logging in case it was requested at the command line.
544 544 """
545 545 if self.logappend:
546 546 self.magic_logstart(self.logappend + ' append')
547 547 elif self.logfile:
548 548 self.magic_logstart(self.logfile)
549 549 elif self.logstart:
550 550 self.magic_logstart()
551 551
552 552 def init_builtins(self):
553 553 self.builtin_trap = BuiltinTrap(shell=self)
554 554
555 555 def init_inspector(self):
556 556 # Object inspector
557 557 self.inspector = oinspect.Inspector(oinspect.InspectColors,
558 558 PyColorize.ANSICodeColors,
559 559 'NoColor',
560 560 self.object_info_string_level)
561 561
562 562 def init_io(self):
563 563 # This will just use sys.stdout and sys.stderr. If you want to
564 564 # override sys.stdout and sys.stderr themselves, you need to do that
565 565 # *before* instantiating this class, because io holds onto
566 566 # references to the underlying streams.
567 567 if sys.platform == 'win32' and self.has_readline:
568 568 io.stdout = io.stderr = io.IOStream(self.readline._outputfile)
569 569 else:
570 570 io.stdout = io.IOStream(sys.stdout)
571 571 io.stderr = io.IOStream(sys.stderr)
572 572
573 573 def init_prompts(self):
574 574 # TODO: This is a pass for now because the prompts are managed inside
575 575 # the DisplayHook. Once there is a separate prompt manager, this
576 576 # will initialize that object and all prompt related information.
577 577 pass
578 578
579 579 def init_display_formatter(self):
580 580 self.display_formatter = DisplayFormatter(config=self.config)
581 581
582 582 def init_display_pub(self):
583 583 self.display_pub = self.display_pub_class(config=self.config)
584 584
585 585 def init_displayhook(self):
586 586 # Initialize displayhook, set in/out prompts and printing system
587 587 self.displayhook = self.displayhook_class(
588 588 config=self.config,
589 589 shell=self,
590 590 cache_size=self.cache_size,
591 591 input_sep = self.separate_in,
592 592 output_sep = self.separate_out,
593 593 output_sep2 = self.separate_out2,
594 594 ps1 = self.prompt_in1,
595 595 ps2 = self.prompt_in2,
596 596 ps_out = self.prompt_out,
597 597 pad_left = self.prompts_pad_left
598 598 )
599 599 # This is a context manager that installs/revmoes the displayhook at
600 600 # the appropriate time.
601 601 self.display_trap = DisplayTrap(hook=self.displayhook)
602 602
603 603 def init_reload_doctest(self):
604 604 # Do a proper resetting of doctest, including the necessary displayhook
605 605 # monkeypatching
606 606 try:
607 607 doctest_reload()
608 608 except ImportError:
609 609 warn("doctest module does not exist.")
610 610
611 611 #-------------------------------------------------------------------------
612 612 # Things related to injections into the sys module
613 613 #-------------------------------------------------------------------------
614 614
615 615 def save_sys_module_state(self):
616 616 """Save the state of hooks in the sys module.
617 617
618 618 This has to be called after self.user_ns is created.
619 619 """
620 620 self._orig_sys_module_state = {}
621 621 self._orig_sys_module_state['stdin'] = sys.stdin
622 622 self._orig_sys_module_state['stdout'] = sys.stdout
623 623 self._orig_sys_module_state['stderr'] = sys.stderr
624 624 self._orig_sys_module_state['excepthook'] = sys.excepthook
625 625 try:
626 626 self._orig_sys_modules_main_name = self.user_ns['__name__']
627 627 except KeyError:
628 628 pass
629 629
630 630 def restore_sys_module_state(self):
631 631 """Restore the state of the sys module."""
632 632 try:
633 633 for k, v in self._orig_sys_module_state.iteritems():
634 634 setattr(sys, k, v)
635 635 except AttributeError:
636 636 pass
637 637 # Reset what what done in self.init_sys_modules
638 638 try:
639 639 sys.modules[self.user_ns['__name__']] = self._orig_sys_modules_main_name
640 640 except (AttributeError, KeyError):
641 641 pass
642 642
643 643 #-------------------------------------------------------------------------
644 644 # Things related to hooks
645 645 #-------------------------------------------------------------------------
646 646
647 647 def init_hooks(self):
648 648 # hooks holds pointers used for user-side customizations
649 649 self.hooks = Struct()
650 650
651 651 self.strdispatchers = {}
652 652
653 653 # Set all default hooks, defined in the IPython.hooks module.
654 654 hooks = IPython.core.hooks
655 655 for hook_name in hooks.__all__:
656 656 # default hooks have priority 100, i.e. low; user hooks should have
657 657 # 0-100 priority
658 658 self.set_hook(hook_name,getattr(hooks,hook_name), 100)
659 659
660 660 def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
661 661 """set_hook(name,hook) -> sets an internal IPython hook.
662 662
663 663 IPython exposes some of its internal API as user-modifiable hooks. By
664 664 adding your function to one of these hooks, you can modify IPython's
665 665 behavior to call at runtime your own routines."""
666 666
667 667 # At some point in the future, this should validate the hook before it
668 668 # accepts it. Probably at least check that the hook takes the number
669 669 # of args it's supposed to.
670 670
671 671 f = types.MethodType(hook,self)
672 672
673 673 # check if the hook is for strdispatcher first
674 674 if str_key is not None:
675 675 sdp = self.strdispatchers.get(name, StrDispatch())
676 676 sdp.add_s(str_key, f, priority )
677 677 self.strdispatchers[name] = sdp
678 678 return
679 679 if re_key is not None:
680 680 sdp = self.strdispatchers.get(name, StrDispatch())
681 681 sdp.add_re(re.compile(re_key), f, priority )
682 682 self.strdispatchers[name] = sdp
683 683 return
684 684
685 685 dp = getattr(self.hooks, name, None)
686 686 if name not in IPython.core.hooks.__all__:
687 687 print "Warning! Hook '%s' is not one of %s" % \
688 688 (name, IPython.core.hooks.__all__ )
689 689 if not dp:
690 690 dp = IPython.core.hooks.CommandChainDispatcher()
691 691
692 692 try:
693 693 dp.add(f,priority)
694 694 except AttributeError:
695 695 # it was not commandchain, plain old func - replace
696 696 dp = f
697 697
698 698 setattr(self.hooks,name, dp)
699 699
700 700 def register_post_execute(self, func):
701 701 """Register a function for calling after code execution.
702 702 """
703 703 if not callable(func):
704 704 raise ValueError('argument %s must be callable' % func)
705 705 self._post_execute[func] = True
706 706
707 707 #-------------------------------------------------------------------------
708 708 # Things related to the "main" module
709 709 #-------------------------------------------------------------------------
710 710
711 711 def new_main_mod(self,ns=None):
712 712 """Return a new 'main' module object for user code execution.
713 713 """
714 714 main_mod = self._user_main_module
715 715 init_fakemod_dict(main_mod,ns)
716 716 return main_mod
717 717
718 718 def cache_main_mod(self,ns,fname):
719 719 """Cache a main module's namespace.
720 720
721 721 When scripts are executed via %run, we must keep a reference to the
722 722 namespace of their __main__ module (a FakeModule instance) around so
723 723 that Python doesn't clear it, rendering objects defined therein
724 724 useless.
725 725
726 726 This method keeps said reference in a private dict, keyed by the
727 727 absolute path of the module object (which corresponds to the script
728 728 path). This way, for multiple executions of the same script we only
729 729 keep one copy of the namespace (the last one), thus preventing memory
730 730 leaks from old references while allowing the objects from the last
731 731 execution to be accessible.
732 732
733 733 Note: we can not allow the actual FakeModule instances to be deleted,
734 734 because of how Python tears down modules (it hard-sets all their
735 735 references to None without regard for reference counts). This method
736 736 must therefore make a *copy* of the given namespace, to allow the
737 737 original module's __dict__ to be cleared and reused.
738 738
739 739
740 740 Parameters
741 741 ----------
742 742 ns : a namespace (a dict, typically)
743 743
744 744 fname : str
745 745 Filename associated with the namespace.
746 746
747 747 Examples
748 748 --------
749 749
750 750 In [10]: import IPython
751 751
752 752 In [11]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
753 753
754 754 In [12]: IPython.__file__ in _ip._main_ns_cache
755 755 Out[12]: True
756 756 """
757 757 self._main_ns_cache[os.path.abspath(fname)] = ns.copy()
758 758
759 759 def clear_main_mod_cache(self):
760 760 """Clear the cache of main modules.
761 761
762 762 Mainly for use by utilities like %reset.
763 763
764 764 Examples
765 765 --------
766 766
767 767 In [15]: import IPython
768 768
769 769 In [16]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
770 770
771 771 In [17]: len(_ip._main_ns_cache) > 0
772 772 Out[17]: True
773 773
774 774 In [18]: _ip.clear_main_mod_cache()
775 775
776 776 In [19]: len(_ip._main_ns_cache) == 0
777 777 Out[19]: True
778 778 """
779 779 self._main_ns_cache.clear()
780 780
781 781 #-------------------------------------------------------------------------
782 782 # Things related to debugging
783 783 #-------------------------------------------------------------------------
784 784
785 785 def init_pdb(self):
786 786 # Set calling of pdb on exceptions
787 787 # self.call_pdb is a property
788 788 self.call_pdb = self.pdb
789 789
790 790 def _get_call_pdb(self):
791 791 return self._call_pdb
792 792
793 793 def _set_call_pdb(self,val):
794 794
795 795 if val not in (0,1,False,True):
796 796 raise ValueError,'new call_pdb value must be boolean'
797 797
798 798 # store value in instance
799 799 self._call_pdb = val
800 800
801 801 # notify the actual exception handlers
802 802 self.InteractiveTB.call_pdb = val
803 803
804 804 call_pdb = property(_get_call_pdb,_set_call_pdb,None,
805 805 'Control auto-activation of pdb at exceptions')
806 806
807 807 def debugger(self,force=False):
808 808 """Call the pydb/pdb debugger.
809 809
810 810 Keywords:
811 811
812 812 - force(False): by default, this routine checks the instance call_pdb
813 813 flag and does not actually invoke the debugger if the flag is false.
814 814 The 'force' option forces the debugger to activate even if the flag
815 815 is false.
816 816 """
817 817
818 818 if not (force or self.call_pdb):
819 819 return
820 820
821 821 if not hasattr(sys,'last_traceback'):
822 822 error('No traceback has been produced, nothing to debug.')
823 823 return
824 824
825 825 # use pydb if available
826 826 if debugger.has_pydb:
827 827 from pydb import pm
828 828 else:
829 829 # fallback to our internal debugger
830 830 pm = lambda : self.InteractiveTB.debugger(force=True)
831 831
832 832 with self.readline_no_record:
833 833 pm()
834 834
835 835 #-------------------------------------------------------------------------
836 836 # Things related to IPython's various namespaces
837 837 #-------------------------------------------------------------------------
838 838
839 839 def init_create_namespaces(self, user_ns=None, user_global_ns=None):
840 840 # Create the namespace where the user will operate. user_ns is
841 841 # normally the only one used, and it is passed to the exec calls as
842 842 # the locals argument. But we do carry a user_global_ns namespace
843 843 # given as the exec 'globals' argument, This is useful in embedding
844 844 # situations where the ipython shell opens in a context where the
845 845 # distinction between locals and globals is meaningful. For
846 846 # non-embedded contexts, it is just the same object as the user_ns dict.
847 847
848 848 # FIXME. For some strange reason, __builtins__ is showing up at user
849 849 # level as a dict instead of a module. This is a manual fix, but I
850 850 # should really track down where the problem is coming from. Alex
851 851 # Schmolck reported this problem first.
852 852
853 853 # A useful post by Alex Martelli on this topic:
854 854 # Re: inconsistent value from __builtins__
855 855 # Von: Alex Martelli <aleaxit@yahoo.com>
856 856 # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
857 857 # Gruppen: comp.lang.python
858 858
859 859 # Michael Hohn <hohn@hooknose.lbl.gov> wrote:
860 860 # > >>> print type(builtin_check.get_global_binding('__builtins__'))
861 861 # > <type 'dict'>
862 862 # > >>> print type(__builtins__)
863 863 # > <type 'module'>
864 864 # > Is this difference in return value intentional?
865 865
866 866 # Well, it's documented that '__builtins__' can be either a dictionary
867 867 # or a module, and it's been that way for a long time. Whether it's
868 868 # intentional (or sensible), I don't know. In any case, the idea is
869 869 # that if you need to access the built-in namespace directly, you
870 870 # should start with "import __builtin__" (note, no 's') which will
871 871 # definitely give you a module. Yeah, it's somewhat confusing:-(.
872 872
873 873 # These routines return properly built dicts as needed by the rest of
874 874 # the code, and can also be used by extension writers to generate
875 875 # properly initialized namespaces.
876 876 user_ns, user_global_ns = self.make_user_namespaces(user_ns,
877 877 user_global_ns)
878 878
879 879 # Assign namespaces
880 880 # This is the namespace where all normal user variables live
881 881 self.user_ns = user_ns
882 882 self.user_global_ns = user_global_ns
883 883
884 884 # An auxiliary namespace that checks what parts of the user_ns were
885 885 # loaded at startup, so we can list later only variables defined in
886 886 # actual interactive use. Since it is always a subset of user_ns, it
887 887 # doesn't need to be separately tracked in the ns_table.
888 888 self.user_ns_hidden = {}
889 889
890 890 # A namespace to keep track of internal data structures to prevent
891 891 # them from cluttering user-visible stuff. Will be updated later
892 892 self.internal_ns = {}
893 893
894 894 # Now that FakeModule produces a real module, we've run into a nasty
895 895 # problem: after script execution (via %run), the module where the user
896 896 # code ran is deleted. Now that this object is a true module (needed
897 897 # so docetst and other tools work correctly), the Python module
898 898 # teardown mechanism runs over it, and sets to None every variable
899 899 # present in that module. Top-level references to objects from the
900 900 # script survive, because the user_ns is updated with them. However,
901 901 # calling functions defined in the script that use other things from
902 902 # the script will fail, because the function's closure had references
903 903 # to the original objects, which are now all None. So we must protect
904 904 # these modules from deletion by keeping a cache.
905 905 #
906 906 # To avoid keeping stale modules around (we only need the one from the
907 907 # last run), we use a dict keyed with the full path to the script, so
908 908 # only the last version of the module is held in the cache. Note,
909 909 # however, that we must cache the module *namespace contents* (their
910 910 # __dict__). Because if we try to cache the actual modules, old ones
911 911 # (uncached) could be destroyed while still holding references (such as
912 912 # those held by GUI objects that tend to be long-lived)>
913 913 #
914 914 # The %reset command will flush this cache. See the cache_main_mod()
915 915 # and clear_main_mod_cache() methods for details on use.
916 916
917 917 # This is the cache used for 'main' namespaces
918 918 self._main_ns_cache = {}
919 919 # And this is the single instance of FakeModule whose __dict__ we keep
920 920 # copying and clearing for reuse on each %run
921 921 self._user_main_module = FakeModule()
922 922
923 923 # A table holding all the namespaces IPython deals with, so that
924 924 # introspection facilities can search easily.
925 925 self.ns_table = {'user':user_ns,
926 926 'user_global':user_global_ns,
927 927 'internal':self.internal_ns,
928 928 'builtin':__builtin__.__dict__
929 929 }
930 930
931 931 # Similarly, track all namespaces where references can be held and that
932 932 # we can safely clear (so it can NOT include builtin). This one can be
933 933 # a simple list. Note that the main execution namespaces, user_ns and
934 934 # user_global_ns, can NOT be listed here, as clearing them blindly
935 935 # causes errors in object __del__ methods. Instead, the reset() method
936 936 # clears them manually and carefully.
937 937 self.ns_refs_table = [ self.user_ns_hidden,
938 938 self.internal_ns, self._main_ns_cache ]
939 939
940 940 def make_user_namespaces(self, user_ns=None, user_global_ns=None):
941 941 """Return a valid local and global user interactive namespaces.
942 942
943 943 This builds a dict with the minimal information needed to operate as a
944 944 valid IPython user namespace, which you can pass to the various
945 945 embedding classes in ipython. The default implementation returns the
946 946 same dict for both the locals and the globals to allow functions to
947 947 refer to variables in the namespace. Customized implementations can
948 948 return different dicts. The locals dictionary can actually be anything
949 949 following the basic mapping protocol of a dict, but the globals dict
950 950 must be a true dict, not even a subclass. It is recommended that any
951 951 custom object for the locals namespace synchronize with the globals
952 952 dict somehow.
953 953
954 954 Raises TypeError if the provided globals namespace is not a true dict.
955 955
956 956 Parameters
957 957 ----------
958 958 user_ns : dict-like, optional
959 959 The current user namespace. The items in this namespace should
960 960 be included in the output. If None, an appropriate blank
961 961 namespace should be created.
962 962 user_global_ns : dict, optional
963 963 The current user global namespace. The items in this namespace
964 964 should be included in the output. If None, an appropriate
965 965 blank namespace should be created.
966 966
967 967 Returns
968 968 -------
969 969 A pair of dictionary-like object to be used as the local namespace
970 970 of the interpreter and a dict to be used as the global namespace.
971 971 """
972 972
973 973
974 974 # We must ensure that __builtin__ (without the final 's') is always
975 975 # available and pointing to the __builtin__ *module*. For more details:
976 976 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
977 977
978 978 if user_ns is None:
979 979 # Set __name__ to __main__ to better match the behavior of the
980 980 # normal interpreter.
981 981 user_ns = {'__name__' :'__main__',
982 982 '__builtin__' : __builtin__,
983 983 '__builtins__' : __builtin__,
984 984 }
985 985 else:
986 986 user_ns.setdefault('__name__','__main__')
987 987 user_ns.setdefault('__builtin__',__builtin__)
988 988 user_ns.setdefault('__builtins__',__builtin__)
989 989
990 990 if user_global_ns is None:
991 991 user_global_ns = user_ns
992 992 if type(user_global_ns) is not dict:
993 993 raise TypeError("user_global_ns must be a true dict; got %r"
994 994 % type(user_global_ns))
995 995
996 996 return user_ns, user_global_ns
997 997
998 998 def init_sys_modules(self):
999 999 # We need to insert into sys.modules something that looks like a
1000 1000 # module but which accesses the IPython namespace, for shelve and
1001 1001 # pickle to work interactively. Normally they rely on getting
1002 1002 # everything out of __main__, but for embedding purposes each IPython
1003 1003 # instance has its own private namespace, so we can't go shoving
1004 1004 # everything into __main__.
1005 1005
1006 1006 # note, however, that we should only do this for non-embedded
1007 1007 # ipythons, which really mimic the __main__.__dict__ with their own
1008 1008 # namespace. Embedded instances, on the other hand, should not do
1009 1009 # this because they need to manage the user local/global namespaces
1010 1010 # only, but they live within a 'normal' __main__ (meaning, they
1011 1011 # shouldn't overtake the execution environment of the script they're
1012 1012 # embedded in).
1013 1013
1014 1014 # This is overridden in the InteractiveShellEmbed subclass to a no-op.
1015 1015
1016 1016 try:
1017 1017 main_name = self.user_ns['__name__']
1018 1018 except KeyError:
1019 1019 raise KeyError('user_ns dictionary MUST have a "__name__" key')
1020 1020 else:
1021 1021 sys.modules[main_name] = FakeModule(self.user_ns)
1022 1022
1023 1023 def init_user_ns(self):
1024 1024 """Initialize all user-visible namespaces to their minimum defaults.
1025 1025
1026 1026 Certain history lists are also initialized here, as they effectively
1027 1027 act as user namespaces.
1028 1028
1029 1029 Notes
1030 1030 -----
1031 1031 All data structures here are only filled in, they are NOT reset by this
1032 1032 method. If they were not empty before, data will simply be added to
1033 1033 therm.
1034 1034 """
1035 1035 # This function works in two parts: first we put a few things in
1036 1036 # user_ns, and we sync that contents into user_ns_hidden so that these
1037 1037 # initial variables aren't shown by %who. After the sync, we add the
1038 1038 # rest of what we *do* want the user to see with %who even on a new
1039 1039 # session (probably nothing, so theye really only see their own stuff)
1040 1040
1041 1041 # The user dict must *always* have a __builtin__ reference to the
1042 1042 # Python standard __builtin__ namespace, which must be imported.
1043 1043 # This is so that certain operations in prompt evaluation can be
1044 1044 # reliably executed with builtins. Note that we can NOT use
1045 1045 # __builtins__ (note the 's'), because that can either be a dict or a
1046 1046 # module, and can even mutate at runtime, depending on the context
1047 1047 # (Python makes no guarantees on it). In contrast, __builtin__ is
1048 1048 # always a module object, though it must be explicitly imported.
1049 1049
1050 1050 # For more details:
1051 1051 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1052 1052 ns = dict(__builtin__ = __builtin__)
1053 1053
1054 1054 # Put 'help' in the user namespace
1055 1055 try:
1056 1056 from site import _Helper
1057 1057 ns['help'] = _Helper()
1058 1058 except ImportError:
1059 1059 warn('help() not available - check site.py')
1060 1060
1061 1061 # make global variables for user access to the histories
1062 1062 ns['_ih'] = self.history_manager.input_hist_parsed
1063 1063 ns['_oh'] = self.history_manager.output_hist
1064 1064 ns['_dh'] = self.history_manager.dir_hist
1065 1065
1066 1066 ns['_sh'] = shadowns
1067 1067
1068 1068 # user aliases to input and output histories. These shouldn't show up
1069 1069 # in %who, as they can have very large reprs.
1070 1070 ns['In'] = self.history_manager.input_hist_parsed
1071 1071 ns['Out'] = self.history_manager.output_hist
1072 1072
1073 1073 # Store myself as the public api!!!
1074 1074 ns['get_ipython'] = self.get_ipython
1075 1075
1076 1076 ns['exit'] = self.exiter
1077 1077 ns['quit'] = self.exiter
1078 1078
1079 1079 # Sync what we've added so far to user_ns_hidden so these aren't seen
1080 1080 # by %who
1081 1081 self.user_ns_hidden.update(ns)
1082 1082
1083 1083 # Anything put into ns now would show up in %who. Think twice before
1084 1084 # putting anything here, as we really want %who to show the user their
1085 1085 # stuff, not our variables.
1086 1086
1087 1087 # Finally, update the real user's namespace
1088 1088 self.user_ns.update(ns)
1089 1089
1090 1090 def reset(self, new_session=True):
1091 1091 """Clear all internal namespaces, and attempt to release references to
1092 1092 user objects.
1093 1093
1094 1094 If new_session is True, a new history session will be opened.
1095 1095 """
1096 1096 # Clear histories
1097 1097 self.history_manager.reset(new_session)
1098 1098 # Reset counter used to index all histories
1099 1099 if new_session:
1100 1100 self.execution_count = 1
1101 1101
1102 1102 # Flush cached output items
1103 1103 if self.displayhook.do_full_cache:
1104 1104 self.displayhook.flush()
1105 1105
1106 1106 # Restore the user namespaces to minimal usability
1107 1107 for ns in self.ns_refs_table:
1108 1108 ns.clear()
1109 1109
1110 1110 # The main execution namespaces must be cleared very carefully,
1111 1111 # skipping the deletion of the builtin-related keys, because doing so
1112 1112 # would cause errors in many object's __del__ methods.
1113 1113 for ns in [self.user_ns, self.user_global_ns]:
1114 1114 drop_keys = set(ns.keys())
1115 1115 drop_keys.discard('__builtin__')
1116 1116 drop_keys.discard('__builtins__')
1117 1117 for k in drop_keys:
1118 1118 del ns[k]
1119 1119
1120 1120 # Restore the user namespaces to minimal usability
1121 1121 self.init_user_ns()
1122 1122
1123 1123 # Restore the default and user aliases
1124 1124 self.alias_manager.clear_aliases()
1125 1125 self.alias_manager.init_aliases()
1126 1126
1127 1127 # Flush the private list of module references kept for script
1128 1128 # execution protection
1129 1129 self.clear_main_mod_cache()
1130 1130
1131 1131 # Clear out the namespace from the last %run
1132 1132 self.new_main_mod()
1133 1133
1134 1134 def del_var(self, varname, by_name=False):
1135 1135 """Delete a variable from the various namespaces, so that, as
1136 1136 far as possible, we're not keeping any hidden references to it.
1137 1137
1138 1138 Parameters
1139 1139 ----------
1140 1140 varname : str
1141 1141 The name of the variable to delete.
1142 1142 by_name : bool
1143 1143 If True, delete variables with the given name in each
1144 1144 namespace. If False (default), find the variable in the user
1145 1145 namespace, and delete references to it.
1146 1146 """
1147 1147 if varname in ('__builtin__', '__builtins__'):
1148 1148 raise ValueError("Refusing to delete %s" % varname)
1149 1149 ns_refs = self.ns_refs_table + [self.user_ns,
1150 1150 self.user_global_ns, self._user_main_module.__dict__] +\
1151 1151 self._main_ns_cache.values()
1152 1152
1153 1153 if by_name: # Delete by name
1154 1154 for ns in ns_refs:
1155 1155 try:
1156 1156 del ns[varname]
1157 1157 except KeyError:
1158 1158 pass
1159 1159 else: # Delete by object
1160 1160 try:
1161 1161 obj = self.user_ns[varname]
1162 1162 except KeyError:
1163 1163 raise NameError("name '%s' is not defined" % varname)
1164 1164 # Also check in output history
1165 1165 ns_refs.append(self.history_manager.output_hist)
1166 1166 for ns in ns_refs:
1167 1167 to_delete = [n for n, o in ns.iteritems() if o is obj]
1168 1168 for name in to_delete:
1169 1169 del ns[name]
1170 1170
1171 1171 # displayhook keeps extra references, but not in a dictionary
1172 1172 for name in ('_', '__', '___'):
1173 1173 if getattr(self.displayhook, name) is obj:
1174 1174 setattr(self.displayhook, name, None)
1175 1175
1176 1176 def reset_selective(self, regex=None):
1177 1177 """Clear selective variables from internal namespaces based on a
1178 1178 specified regular expression.
1179 1179
1180 1180 Parameters
1181 1181 ----------
1182 1182 regex : string or compiled pattern, optional
1183 1183 A regular expression pattern that will be used in searching
1184 1184 variable names in the users namespaces.
1185 1185 """
1186 1186 if regex is not None:
1187 1187 try:
1188 1188 m = re.compile(regex)
1189 1189 except TypeError:
1190 1190 raise TypeError('regex must be a string or compiled pattern')
1191 1191 # Search for keys in each namespace that match the given regex
1192 1192 # If a match is found, delete the key/value pair.
1193 1193 for ns in self.ns_refs_table:
1194 1194 for var in ns:
1195 1195 if m.search(var):
1196 1196 del ns[var]
1197 1197
1198 1198 def push(self, variables, interactive=True):
1199 1199 """Inject a group of variables into the IPython user namespace.
1200 1200
1201 1201 Parameters
1202 1202 ----------
1203 1203 variables : dict, str or list/tuple of str
1204 1204 The variables to inject into the user's namespace. If a dict, a
1205 1205 simple update is done. If a str, the string is assumed to have
1206 1206 variable names separated by spaces. A list/tuple of str can also
1207 1207 be used to give the variable names. If just the variable names are
1208 1208 give (list/tuple/str) then the variable values looked up in the
1209 1209 callers frame.
1210 1210 interactive : bool
1211 1211 If True (default), the variables will be listed with the ``who``
1212 1212 magic.
1213 1213 """
1214 1214 vdict = None
1215 1215
1216 1216 # We need a dict of name/value pairs to do namespace updates.
1217 1217 if isinstance(variables, dict):
1218 1218 vdict = variables
1219 1219 elif isinstance(variables, (basestring, list, tuple)):
1220 1220 if isinstance(variables, basestring):
1221 1221 vlist = variables.split()
1222 1222 else:
1223 1223 vlist = variables
1224 1224 vdict = {}
1225 1225 cf = sys._getframe(1)
1226 1226 for name in vlist:
1227 1227 try:
1228 1228 vdict[name] = eval(name, cf.f_globals, cf.f_locals)
1229 1229 except:
1230 1230 print ('Could not get variable %s from %s' %
1231 1231 (name,cf.f_code.co_name))
1232 1232 else:
1233 1233 raise ValueError('variables must be a dict/str/list/tuple')
1234 1234
1235 1235 # Propagate variables to user namespace
1236 1236 self.user_ns.update(vdict)
1237 1237
1238 1238 # And configure interactive visibility
1239 1239 config_ns = self.user_ns_hidden
1240 1240 if interactive:
1241 1241 for name, val in vdict.iteritems():
1242 1242 config_ns.pop(name, None)
1243 1243 else:
1244 1244 for name,val in vdict.iteritems():
1245 1245 config_ns[name] = val
1246 1246
1247 1247 #-------------------------------------------------------------------------
1248 1248 # Things related to object introspection
1249 1249 #-------------------------------------------------------------------------
1250 1250
1251 1251 def _ofind(self, oname, namespaces=None):
1252 1252 """Find an object in the available namespaces.
1253 1253
1254 1254 self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
1255 1255
1256 1256 Has special code to detect magic functions.
1257 1257 """
1258 1258 #oname = oname.strip()
1259 1259 #print '1- oname: <%r>' % oname # dbg
1260 1260 try:
1261 1261 oname = oname.strip().encode('ascii')
1262 1262 #print '2- oname: <%r>' % oname # dbg
1263 1263 except UnicodeEncodeError:
1264 1264 print 'Python identifiers can only contain ascii characters.'
1265 1265 return dict(found=False)
1266 1266
1267 1267 alias_ns = None
1268 1268 if namespaces is None:
1269 1269 # Namespaces to search in:
1270 1270 # Put them in a list. The order is important so that we
1271 1271 # find things in the same order that Python finds them.
1272 1272 namespaces = [ ('Interactive', self.user_ns),
1273 1273 ('IPython internal', self.internal_ns),
1274 1274 ('Python builtin', __builtin__.__dict__),
1275 1275 ('Alias', self.alias_manager.alias_table),
1276 1276 ]
1277 1277 alias_ns = self.alias_manager.alias_table
1278 1278
1279 1279 # initialize results to 'null'
1280 1280 found = False; obj = None; ospace = None; ds = None;
1281 1281 ismagic = False; isalias = False; parent = None
1282 1282
1283 1283 # We need to special-case 'print', which as of python2.6 registers as a
1284 1284 # function but should only be treated as one if print_function was
1285 1285 # loaded with a future import. In this case, just bail.
1286 1286 if (oname == 'print' and not (self.compile.compiler_flags &
1287 1287 __future__.CO_FUTURE_PRINT_FUNCTION)):
1288 1288 return {'found':found, 'obj':obj, 'namespace':ospace,
1289 1289 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1290 1290
1291 1291 # Look for the given name by splitting it in parts. If the head is
1292 1292 # found, then we look for all the remaining parts as members, and only
1293 1293 # declare success if we can find them all.
1294 1294 oname_parts = oname.split('.')
1295 1295 oname_head, oname_rest = oname_parts[0],oname_parts[1:]
1296 1296 for nsname,ns in namespaces:
1297 1297 try:
1298 1298 obj = ns[oname_head]
1299 1299 except KeyError:
1300 1300 continue
1301 1301 else:
1302 1302 #print 'oname_rest:', oname_rest # dbg
1303 1303 for part in oname_rest:
1304 1304 try:
1305 1305 parent = obj
1306 1306 obj = getattr(obj,part)
1307 1307 except:
1308 1308 # Blanket except b/c some badly implemented objects
1309 1309 # allow __getattr__ to raise exceptions other than
1310 1310 # AttributeError, which then crashes IPython.
1311 1311 break
1312 1312 else:
1313 1313 # If we finish the for loop (no break), we got all members
1314 1314 found = True
1315 1315 ospace = nsname
1316 1316 if ns == alias_ns:
1317 1317 isalias = True
1318 1318 break # namespace loop
1319 1319
1320 1320 # Try to see if it's magic
1321 1321 if not found:
1322 1322 if oname.startswith(ESC_MAGIC):
1323 1323 oname = oname[1:]
1324 1324 obj = getattr(self,'magic_'+oname,None)
1325 1325 if obj is not None:
1326 1326 found = True
1327 1327 ospace = 'IPython internal'
1328 1328 ismagic = True
1329 1329
1330 1330 # Last try: special-case some literals like '', [], {}, etc:
1331 1331 if not found and oname_head in ["''",'""','[]','{}','()']:
1332 1332 obj = eval(oname_head)
1333 1333 found = True
1334 1334 ospace = 'Interactive'
1335 1335
1336 1336 return {'found':found, 'obj':obj, 'namespace':ospace,
1337 1337 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1338 1338
1339 1339 def _ofind_property(self, oname, info):
1340 1340 """Second part of object finding, to look for property details."""
1341 1341 if info.found:
1342 1342 # Get the docstring of the class property if it exists.
1343 1343 path = oname.split('.')
1344 1344 root = '.'.join(path[:-1])
1345 1345 if info.parent is not None:
1346 1346 try:
1347 1347 target = getattr(info.parent, '__class__')
1348 1348 # The object belongs to a class instance.
1349 1349 try:
1350 1350 target = getattr(target, path[-1])
1351 1351 # The class defines the object.
1352 1352 if isinstance(target, property):
1353 1353 oname = root + '.__class__.' + path[-1]
1354 1354 info = Struct(self._ofind(oname))
1355 1355 except AttributeError: pass
1356 1356 except AttributeError: pass
1357 1357
1358 1358 # We return either the new info or the unmodified input if the object
1359 1359 # hadn't been found
1360 1360 return info
1361 1361
1362 1362 def _object_find(self, oname, namespaces=None):
1363 1363 """Find an object and return a struct with info about it."""
1364 1364 inf = Struct(self._ofind(oname, namespaces))
1365 1365 return Struct(self._ofind_property(oname, inf))
1366 1366
1367 1367 def _inspect(self, meth, oname, namespaces=None, **kw):
1368 1368 """Generic interface to the inspector system.
1369 1369
1370 1370 This function is meant to be called by pdef, pdoc & friends."""
1371 1371 info = self._object_find(oname)
1372 1372 if info.found:
1373 1373 pmethod = getattr(self.inspector, meth)
1374 1374 formatter = format_screen if info.ismagic else None
1375 1375 if meth == 'pdoc':
1376 1376 pmethod(info.obj, oname, formatter)
1377 1377 elif meth == 'pinfo':
1378 1378 pmethod(info.obj, oname, formatter, info, **kw)
1379 1379 else:
1380 1380 pmethod(info.obj, oname)
1381 1381 else:
1382 1382 print 'Object `%s` not found.' % oname
1383 1383 return 'not found' # so callers can take other action
1384 1384
1385 1385 def object_inspect(self, oname):
1386 1386 with self.builtin_trap:
1387 1387 info = self._object_find(oname)
1388 1388 if info.found:
1389 1389 return self.inspector.info(info.obj, oname, info=info)
1390 1390 else:
1391 1391 return oinspect.object_info(name=oname, found=False)
1392 1392
1393 1393 #-------------------------------------------------------------------------
1394 1394 # Things related to history management
1395 1395 #-------------------------------------------------------------------------
1396 1396
1397 1397 def init_history(self):
1398 1398 """Sets up the command history, and starts regular autosaves."""
1399 1399 self.history_manager = HistoryManager(shell=self, config=self.config)
1400 1400
1401 1401 #-------------------------------------------------------------------------
1402 1402 # Things related to exception handling and tracebacks (not debugging)
1403 1403 #-------------------------------------------------------------------------
1404 1404
1405 1405 def init_traceback_handlers(self, custom_exceptions):
1406 1406 # Syntax error handler.
1407 1407 self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor')
1408 1408
1409 1409 # The interactive one is initialized with an offset, meaning we always
1410 1410 # want to remove the topmost item in the traceback, which is our own
1411 1411 # internal code. Valid modes: ['Plain','Context','Verbose']
1412 1412 self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
1413 1413 color_scheme='NoColor',
1414 1414 tb_offset = 1,
1415 1415 check_cache=self.compile.check_cache)
1416 1416
1417 1417 # The instance will store a pointer to the system-wide exception hook,
1418 1418 # so that runtime code (such as magics) can access it. This is because
1419 1419 # during the read-eval loop, it may get temporarily overwritten.
1420 1420 self.sys_excepthook = sys.excepthook
1421 1421
1422 1422 # and add any custom exception handlers the user may have specified
1423 1423 self.set_custom_exc(*custom_exceptions)
1424 1424
1425 1425 # Set the exception mode
1426 1426 self.InteractiveTB.set_mode(mode=self.xmode)
1427 1427
1428 1428 def set_custom_exc(self, exc_tuple, handler):
1429 1429 """set_custom_exc(exc_tuple,handler)
1430 1430
1431 1431 Set a custom exception handler, which will be called if any of the
1432 1432 exceptions in exc_tuple occur in the mainloop (specifically, in the
1433 1433 run_code() method.
1434 1434
1435 1435 Inputs:
1436 1436
1437 1437 - exc_tuple: a *tuple* of valid exceptions to call the defined
1438 1438 handler for. It is very important that you use a tuple, and NOT A
1439 1439 LIST here, because of the way Python's except statement works. If
1440 1440 you only want to trap a single exception, use a singleton tuple:
1441 1441
1442 1442 exc_tuple == (MyCustomException,)
1443 1443
1444 1444 - handler: this must be defined as a function with the following
1445 1445 basic interface::
1446 1446
1447 1447 def my_handler(self, etype, value, tb, tb_offset=None)
1448 1448 ...
1449 1449 # The return value must be
1450 1450 return structured_traceback
1451 1451
1452 1452 This will be made into an instance method (via types.MethodType)
1453 1453 of IPython itself, and it will be called if any of the exceptions
1454 1454 listed in the exc_tuple are caught. If the handler is None, an
1455 1455 internal basic one is used, which just prints basic info.
1456 1456
1457 1457 WARNING: by putting in your own exception handler into IPython's main
1458 1458 execution loop, you run a very good chance of nasty crashes. This
1459 1459 facility should only be used if you really know what you are doing."""
1460 1460
1461 1461 assert type(exc_tuple)==type(()) , \
1462 1462 "The custom exceptions must be given AS A TUPLE."
1463 1463
1464 1464 def dummy_handler(self,etype,value,tb):
1465 1465 print '*** Simple custom exception handler ***'
1466 1466 print 'Exception type :',etype
1467 1467 print 'Exception value:',value
1468 1468 print 'Traceback :',tb
1469 1469 #print 'Source code :','\n'.join(self.buffer)
1470 1470
1471 1471 if handler is None: handler = dummy_handler
1472 1472
1473 1473 self.CustomTB = types.MethodType(handler,self)
1474 1474 self.custom_exceptions = exc_tuple
1475 1475
1476 1476 def excepthook(self, etype, value, tb):
1477 1477 """One more defense for GUI apps that call sys.excepthook.
1478 1478
1479 1479 GUI frameworks like wxPython trap exceptions and call
1480 1480 sys.excepthook themselves. I guess this is a feature that
1481 1481 enables them to keep running after exceptions that would
1482 1482 otherwise kill their mainloop. This is a bother for IPython
1483 1483 which excepts to catch all of the program exceptions with a try:
1484 1484 except: statement.
1485 1485
1486 1486 Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
1487 1487 any app directly invokes sys.excepthook, it will look to the user like
1488 1488 IPython crashed. In order to work around this, we can disable the
1489 1489 CrashHandler and replace it with this excepthook instead, which prints a
1490 1490 regular traceback using our InteractiveTB. In this fashion, apps which
1491 1491 call sys.excepthook will generate a regular-looking exception from
1492 1492 IPython, and the CrashHandler will only be triggered by real IPython
1493 1493 crashes.
1494 1494
1495 1495 This hook should be used sparingly, only in places which are not likely
1496 1496 to be true IPython errors.
1497 1497 """
1498 1498 self.showtraceback((etype,value,tb),tb_offset=0)
1499 1499
1500 1500 def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
1501 1501 exception_only=False):
1502 1502 """Display the exception that just occurred.
1503 1503
1504 1504 If nothing is known about the exception, this is the method which
1505 1505 should be used throughout the code for presenting user tracebacks,
1506 1506 rather than directly invoking the InteractiveTB object.
1507 1507
1508 1508 A specific showsyntaxerror() also exists, but this method can take
1509 1509 care of calling it if needed, so unless you are explicitly catching a
1510 1510 SyntaxError exception, don't try to analyze the stack manually and
1511 1511 simply call this method."""
1512 1512
1513 1513 try:
1514 1514 if exc_tuple is None:
1515 1515 etype, value, tb = sys.exc_info()
1516 1516 else:
1517 1517 etype, value, tb = exc_tuple
1518 1518
1519 1519 if etype is None:
1520 1520 if hasattr(sys, 'last_type'):
1521 1521 etype, value, tb = sys.last_type, sys.last_value, \
1522 1522 sys.last_traceback
1523 1523 else:
1524 1524 self.write_err('No traceback available to show.\n')
1525 1525 return
1526 1526
1527 1527 if etype is SyntaxError:
1528 1528 # Though this won't be called by syntax errors in the input
1529 1529 # line, there may be SyntaxError cases whith imported code.
1530 1530 self.showsyntaxerror(filename)
1531 1531 elif etype is UsageError:
1532 1532 print "UsageError:", value
1533 1533 else:
1534 1534 # WARNING: these variables are somewhat deprecated and not
1535 1535 # necessarily safe to use in a threaded environment, but tools
1536 1536 # like pdb depend on their existence, so let's set them. If we
1537 1537 # find problems in the field, we'll need to revisit their use.
1538 1538 sys.last_type = etype
1539 1539 sys.last_value = value
1540 1540 sys.last_traceback = tb
1541 1541 if etype in self.custom_exceptions:
1542 1542 # FIXME: Old custom traceback objects may just return a
1543 1543 # string, in that case we just put it into a list
1544 1544 stb = self.CustomTB(etype, value, tb, tb_offset)
1545 1545 if isinstance(ctb, basestring):
1546 1546 stb = [stb]
1547 1547 else:
1548 1548 if exception_only:
1549 1549 stb = ['An exception has occurred, use %tb to see '
1550 1550 'the full traceback.\n']
1551 1551 stb.extend(self.InteractiveTB.get_exception_only(etype,
1552 1552 value))
1553 1553 else:
1554 1554 stb = self.InteractiveTB.structured_traceback(etype,
1555 1555 value, tb, tb_offset=tb_offset)
1556 1556
1557 1557 if self.call_pdb:
1558 1558 # drop into debugger
1559 1559 self.debugger(force=True)
1560 1560
1561 1561 # Actually show the traceback
1562 1562 self._showtraceback(etype, value, stb)
1563 1563
1564 1564 except KeyboardInterrupt:
1565 1565 self.write_err("\nKeyboardInterrupt\n")
1566 1566
1567 1567 def _showtraceback(self, etype, evalue, stb):
1568 1568 """Actually show a traceback.
1569 1569
1570 1570 Subclasses may override this method to put the traceback on a different
1571 1571 place, like a side channel.
1572 1572 """
1573 1573 print >> io.stdout, self.InteractiveTB.stb2text(stb)
1574 1574
1575 1575 def showsyntaxerror(self, filename=None):
1576 1576 """Display the syntax error that just occurred.
1577 1577
1578 1578 This doesn't display a stack trace because there isn't one.
1579 1579
1580 1580 If a filename is given, it is stuffed in the exception instead
1581 1581 of what was there before (because Python's parser always uses
1582 1582 "<string>" when reading from a string).
1583 1583 """
1584 1584 etype, value, last_traceback = sys.exc_info()
1585 1585
1586 1586 # See note about these variables in showtraceback() above
1587 1587 sys.last_type = etype
1588 1588 sys.last_value = value
1589 1589 sys.last_traceback = last_traceback
1590 1590
1591 1591 if filename and etype is SyntaxError:
1592 1592 # Work hard to stuff the correct filename in the exception
1593 1593 try:
1594 1594 msg, (dummy_filename, lineno, offset, line) = value
1595 1595 except:
1596 1596 # Not the format we expect; leave it alone
1597 1597 pass
1598 1598 else:
1599 1599 # Stuff in the right filename
1600 1600 try:
1601 1601 # Assume SyntaxError is a class exception
1602 1602 value = SyntaxError(msg, (filename, lineno, offset, line))
1603 1603 except:
1604 1604 # If that failed, assume SyntaxError is a string
1605 1605 value = msg, (filename, lineno, offset, line)
1606 1606 stb = self.SyntaxTB.structured_traceback(etype, value, [])
1607 1607 self._showtraceback(etype, value, stb)
1608 1608
1609 1609 #-------------------------------------------------------------------------
1610 1610 # Things related to readline
1611 1611 #-------------------------------------------------------------------------
1612 1612
1613 1613 def init_readline(self):
1614 1614 """Command history completion/saving/reloading."""
1615 1615
1616 1616 if self.readline_use:
1617 1617 import IPython.utils.rlineimpl as readline
1618 1618
1619 1619 self.rl_next_input = None
1620 1620 self.rl_do_indent = False
1621 1621
1622 1622 if not self.readline_use or not readline.have_readline:
1623 1623 self.has_readline = False
1624 1624 self.readline = None
1625 1625 # Set a number of methods that depend on readline to be no-op
1626 1626 self.set_readline_completer = no_op
1627 1627 self.set_custom_completer = no_op
1628 1628 self.set_completer_frame = no_op
1629 1629 warn('Readline services not available or not loaded.')
1630 1630 else:
1631 1631 self.has_readline = True
1632 1632 self.readline = readline
1633 1633 sys.modules['readline'] = readline
1634 1634
1635 1635 # Platform-specific configuration
1636 1636 if os.name == 'nt':
1637 1637 # FIXME - check with Frederick to see if we can harmonize
1638 1638 # naming conventions with pyreadline to avoid this
1639 1639 # platform-dependent check
1640 1640 self.readline_startup_hook = readline.set_pre_input_hook
1641 1641 else:
1642 1642 self.readline_startup_hook = readline.set_startup_hook
1643 1643
1644 1644 # Load user's initrc file (readline config)
1645 1645 # Or if libedit is used, load editrc.
1646 1646 inputrc_name = os.environ.get('INPUTRC')
1647 1647 if inputrc_name is None:
1648 1648 home_dir = get_home_dir()
1649 1649 if home_dir is not None:
1650 1650 inputrc_name = '.inputrc'
1651 1651 if readline.uses_libedit:
1652 1652 inputrc_name = '.editrc'
1653 1653 inputrc_name = os.path.join(home_dir, inputrc_name)
1654 1654 if os.path.isfile(inputrc_name):
1655 1655 try:
1656 1656 readline.read_init_file(inputrc_name)
1657 1657 except:
1658 1658 warn('Problems reading readline initialization file <%s>'
1659 1659 % inputrc_name)
1660 1660
1661 1661 # Configure readline according to user's prefs
1662 1662 # This is only done if GNU readline is being used. If libedit
1663 1663 # is being used (as on Leopard) the readline config is
1664 1664 # not run as the syntax for libedit is different.
1665 1665 if not readline.uses_libedit:
1666 1666 for rlcommand in self.readline_parse_and_bind:
1667 1667 #print "loading rl:",rlcommand # dbg
1668 1668 readline.parse_and_bind(rlcommand)
1669 1669
1670 1670 # Remove some chars from the delimiters list. If we encounter
1671 1671 # unicode chars, discard them.
1672 1672 delims = readline.get_completer_delims().encode("ascii", "ignore")
1673 1673 delims = delims.translate(None, self.readline_remove_delims)
1674 1674 delims = delims.replace(ESC_MAGIC, '')
1675 1675 readline.set_completer_delims(delims)
1676 1676 # otherwise we end up with a monster history after a while:
1677 1677 readline.set_history_length(self.history_length)
1678 1678
1679 1679 self.refill_readline_hist()
1680 1680 self.readline_no_record = ReadlineNoRecord(self)
1681 1681
1682 1682 # Configure auto-indent for all platforms
1683 1683 self.set_autoindent(self.autoindent)
1684 1684
1685 1685 def refill_readline_hist(self):
1686 1686 # Load the last 1000 lines from history
1687 1687 self.readline.clear_history()
1688 1688 stdin_encoding = sys.stdin.encoding or "utf-8"
1689 1689 for _, _, cell in self.history_manager.get_tail(1000,
1690 1690 include_latest=True):
1691 1691 if cell.strip(): # Ignore blank lines
1692 1692 for line in cell.splitlines():
1693 1693 self.readline.add_history(line.encode(stdin_encoding, 'replace'))
1694 1694
1695 1695 def set_next_input(self, s):
1696 1696 """ Sets the 'default' input string for the next command line.
1697 1697
1698 1698 Requires readline.
1699 1699
1700 1700 Example:
1701 1701
1702 1702 [D:\ipython]|1> _ip.set_next_input("Hello Word")
1703 1703 [D:\ipython]|2> Hello Word_ # cursor is here
1704 1704 """
1705 1705
1706 1706 self.rl_next_input = s
1707 1707
1708 1708 # Maybe move this to the terminal subclass?
1709 1709 def pre_readline(self):
1710 1710 """readline hook to be used at the start of each line.
1711 1711
1712 1712 Currently it handles auto-indent only."""
1713 1713
1714 1714 if self.rl_do_indent:
1715 1715 self.readline.insert_text(self._indent_current_str())
1716 1716 if self.rl_next_input is not None:
1717 1717 self.readline.insert_text(self.rl_next_input)
1718 1718 self.rl_next_input = None
1719 1719
1720 1720 def _indent_current_str(self):
1721 1721 """return the current level of indentation as a string"""
1722 1722 return self.input_splitter.indent_spaces * ' '
1723 1723
1724 1724 #-------------------------------------------------------------------------
1725 1725 # Things related to text completion
1726 1726 #-------------------------------------------------------------------------
1727 1727
1728 1728 def init_completer(self):
1729 1729 """Initialize the completion machinery.
1730 1730
1731 1731 This creates completion machinery that can be used by client code,
1732 1732 either interactively in-process (typically triggered by the readline
1733 1733 library), programatically (such as in test suites) or out-of-prcess
1734 1734 (typically over the network by remote frontends).
1735 1735 """
1736 1736 from IPython.core.completer import IPCompleter
1737 1737 from IPython.core.completerlib import (module_completer,
1738 1738 magic_run_completer, cd_completer)
1739 1739
1740 1740 self.Completer = IPCompleter(self,
1741 1741 self.user_ns,
1742 1742 self.user_global_ns,
1743 1743 self.readline_omit__names,
1744 1744 self.alias_manager.alias_table,
1745 1745 self.has_readline)
1746 1746
1747 1747 # Add custom completers to the basic ones built into IPCompleter
1748 1748 sdisp = self.strdispatchers.get('complete_command', StrDispatch())
1749 1749 self.strdispatchers['complete_command'] = sdisp
1750 1750 self.Completer.custom_completers = sdisp
1751 1751
1752 1752 self.set_hook('complete_command', module_completer, str_key = 'import')
1753 1753 self.set_hook('complete_command', module_completer, str_key = 'from')
1754 1754 self.set_hook('complete_command', magic_run_completer, str_key = '%run')
1755 1755 self.set_hook('complete_command', cd_completer, str_key = '%cd')
1756 1756
1757 1757 # Only configure readline if we truly are using readline. IPython can
1758 1758 # do tab-completion over the network, in GUIs, etc, where readline
1759 1759 # itself may be absent
1760 1760 if self.has_readline:
1761 1761 self.set_readline_completer()
1762 1762
1763 1763 def complete(self, text, line=None, cursor_pos=None):
1764 1764 """Return the completed text and a list of completions.
1765 1765
1766 1766 Parameters
1767 1767 ----------
1768 1768
1769 1769 text : string
1770 1770 A string of text to be completed on. It can be given as empty and
1771 1771 instead a line/position pair are given. In this case, the
1772 1772 completer itself will split the line like readline does.
1773 1773
1774 1774 line : string, optional
1775 1775 The complete line that text is part of.
1776 1776
1777 1777 cursor_pos : int, optional
1778 1778 The position of the cursor on the input line.
1779 1779
1780 1780 Returns
1781 1781 -------
1782 1782 text : string
1783 1783 The actual text that was completed.
1784 1784
1785 1785 matches : list
1786 1786 A sorted list with all possible completions.
1787 1787
1788 1788 The optional arguments allow the completion to take more context into
1789 1789 account, and are part of the low-level completion API.
1790 1790
1791 1791 This is a wrapper around the completion mechanism, similar to what
1792 1792 readline does at the command line when the TAB key is hit. By
1793 1793 exposing it as a method, it can be used by other non-readline
1794 1794 environments (such as GUIs) for text completion.
1795 1795
1796 1796 Simple usage example:
1797 1797
1798 1798 In [1]: x = 'hello'
1799 1799
1800 1800 In [2]: _ip.complete('x.l')
1801 1801 Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
1802 1802 """
1803 1803
1804 1804 # Inject names into __builtin__ so we can complete on the added names.
1805 1805 with self.builtin_trap:
1806 1806 return self.Completer.complete(text, line, cursor_pos)
1807 1807
1808 1808 def set_custom_completer(self, completer, pos=0):
1809 1809 """Adds a new custom completer function.
1810 1810
1811 1811 The position argument (defaults to 0) is the index in the completers
1812 1812 list where you want the completer to be inserted."""
1813 1813
1814 1814 newcomp = types.MethodType(completer,self.Completer)
1815 1815 self.Completer.matchers.insert(pos,newcomp)
1816 1816
1817 1817 def set_readline_completer(self):
1818 1818 """Reset readline's completer to be our own."""
1819 1819 self.readline.set_completer(self.Completer.rlcomplete)
1820 1820
1821 1821 def set_completer_frame(self, frame=None):
1822 1822 """Set the frame of the completer."""
1823 1823 if frame:
1824 1824 self.Completer.namespace = frame.f_locals
1825 1825 self.Completer.global_namespace = frame.f_globals
1826 1826 else:
1827 1827 self.Completer.namespace = self.user_ns
1828 1828 self.Completer.global_namespace = self.user_global_ns
1829 1829
1830 1830 #-------------------------------------------------------------------------
1831 1831 # Things related to magics
1832 1832 #-------------------------------------------------------------------------
1833 1833
1834 1834 def init_magics(self):
1835 1835 # FIXME: Move the color initialization to the DisplayHook, which
1836 1836 # should be split into a prompt manager and displayhook. We probably
1837 1837 # even need a centralize colors management object.
1838 1838 self.magic_colors(self.colors)
1839 1839 # History was moved to a separate module
1840 1840 from . import history
1841 1841 history.init_ipython(self)
1842 1842
1843 1843 def magic(self,arg_s):
1844 1844 """Call a magic function by name.
1845 1845
1846 1846 Input: a string containing the name of the magic function to call and
1847 1847 any additional arguments to be passed to the magic.
1848 1848
1849 1849 magic('name -opt foo bar') is equivalent to typing at the ipython
1850 1850 prompt:
1851 1851
1852 1852 In[1]: %name -opt foo bar
1853 1853
1854 1854 To call a magic without arguments, simply use magic('name').
1855 1855
1856 1856 This provides a proper Python function to call IPython's magics in any
1857 1857 valid Python code you can type at the interpreter, including loops and
1858 1858 compound statements.
1859 1859 """
1860 1860 args = arg_s.split(' ',1)
1861 1861 magic_name = args[0]
1862 1862 magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
1863 1863
1864 1864 try:
1865 1865 magic_args = args[1]
1866 1866 except IndexError:
1867 1867 magic_args = ''
1868 1868 fn = getattr(self,'magic_'+magic_name,None)
1869 1869 if fn is None:
1870 1870 error("Magic function `%s` not found." % magic_name)
1871 1871 else:
1872 1872 magic_args = self.var_expand(magic_args,1)
1873 1873 # Grab local namespace if we need it:
1874 1874 if getattr(fn, "needs_local_scope", False):
1875 1875 self._magic_locals = sys._getframe(1).f_locals
1876 1876 with self.builtin_trap:
1877 1877 result = fn(magic_args)
1878 1878 # Ensure we're not keeping object references around:
1879 1879 self._magic_locals = {}
1880 1880 return result
1881 1881
1882 1882 def define_magic(self, magicname, func):
1883 1883 """Expose own function as magic function for ipython
1884 1884
1885 1885 def foo_impl(self,parameter_s=''):
1886 1886 'My very own magic!. (Use docstrings, IPython reads them).'
1887 1887 print 'Magic function. Passed parameter is between < >:'
1888 1888 print '<%s>' % parameter_s
1889 1889 print 'The self object is:',self
1890 1890
1891 1891 self.define_magic('foo',foo_impl)
1892 1892 """
1893 1893
1894 1894 import new
1895 1895 im = types.MethodType(func,self)
1896 1896 old = getattr(self, "magic_" + magicname, None)
1897 1897 setattr(self, "magic_" + magicname, im)
1898 1898 return old
1899 1899
1900 1900 #-------------------------------------------------------------------------
1901 1901 # Things related to macros
1902 1902 #-------------------------------------------------------------------------
1903 1903
1904 1904 def define_macro(self, name, themacro):
1905 1905 """Define a new macro
1906 1906
1907 1907 Parameters
1908 1908 ----------
1909 1909 name : str
1910 1910 The name of the macro.
1911 1911 themacro : str or Macro
1912 1912 The action to do upon invoking the macro. If a string, a new
1913 1913 Macro object is created by passing the string to it.
1914 1914 """
1915 1915
1916 1916 from IPython.core import macro
1917 1917
1918 1918 if isinstance(themacro, basestring):
1919 1919 themacro = macro.Macro(themacro)
1920 1920 if not isinstance(themacro, macro.Macro):
1921 1921 raise ValueError('A macro must be a string or a Macro instance.')
1922 1922 self.user_ns[name] = themacro
1923 1923
1924 1924 #-------------------------------------------------------------------------
1925 1925 # Things related to the running of system commands
1926 1926 #-------------------------------------------------------------------------
1927 1927
1928 1928 def system_piped(self, cmd):
1929 1929 """Call the given cmd in a subprocess, piping stdout/err
1930 1930
1931 1931 Parameters
1932 1932 ----------
1933 1933 cmd : str
1934 1934 Command to execute (can not end in '&', as background processes are
1935 1935 not supported. Should not be a command that expects input
1936 1936 other than simple text.
1937 1937 """
1938 1938 if cmd.rstrip().endswith('&'):
1939 1939 # this is *far* from a rigorous test
1940 1940 # We do not support backgrounding processes because we either use
1941 1941 # pexpect or pipes to read from. Users can always just call
1942 1942 # os.system() or use ip.system=ip.system_raw
1943 1943 # if they really want a background process.
1944 1944 raise OSError("Background processes not supported.")
1945 1945
1946 1946 # we explicitly do NOT return the subprocess status code, because
1947 1947 # a non-None value would trigger :func:`sys.displayhook` calls.
1948 1948 # Instead, we store the exit_code in user_ns.
1949 1949 self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=2))
1950 1950
1951 1951 def system_raw(self, cmd):
1952 1952 """Call the given cmd in a subprocess using os.system
1953 1953
1954 1954 Parameters
1955 1955 ----------
1956 1956 cmd : str
1957 1957 Command to execute.
1958 1958 """
1959 1959 # We explicitly do NOT return the subprocess status code, because
1960 1960 # a non-None value would trigger :func:`sys.displayhook` calls.
1961 1961 # Instead, we store the exit_code in user_ns.
1962 1962 self.user_ns['_exit_code'] = os.system(self.var_expand(cmd, depth=2))
1963 1963
1964 1964 # use piped system by default, because it is better behaved
1965 1965 system = system_piped
1966 1966
1967 1967 def getoutput(self, cmd, split=True):
1968 1968 """Get output (possibly including stderr) from a subprocess.
1969 1969
1970 1970 Parameters
1971 1971 ----------
1972 1972 cmd : str
1973 1973 Command to execute (can not end in '&', as background processes are
1974 1974 not supported.
1975 1975 split : bool, optional
1976 1976
1977 1977 If True, split the output into an IPython SList. Otherwise, an
1978 1978 IPython LSString is returned. These are objects similar to normal
1979 1979 lists and strings, with a few convenience attributes for easier
1980 1980 manipulation of line-based output. You can use '?' on them for
1981 1981 details.
1982 1982 """
1983 1983 if cmd.rstrip().endswith('&'):
1984 1984 # this is *far* from a rigorous test
1985 1985 raise OSError("Background processes not supported.")
1986 1986 out = getoutput(self.var_expand(cmd, depth=2))
1987 1987 if split:
1988 1988 out = SList(out.splitlines())
1989 1989 else:
1990 1990 out = LSString(out)
1991 1991 return out
1992 1992
1993 1993 #-------------------------------------------------------------------------
1994 1994 # Things related to aliases
1995 1995 #-------------------------------------------------------------------------
1996 1996
1997 1997 def init_alias(self):
1998 1998 self.alias_manager = AliasManager(shell=self, config=self.config)
1999 1999 self.ns_table['alias'] = self.alias_manager.alias_table,
2000 2000
2001 2001 #-------------------------------------------------------------------------
2002 2002 # Things related to extensions and plugins
2003 2003 #-------------------------------------------------------------------------
2004 2004
2005 2005 def init_extension_manager(self):
2006 2006 self.extension_manager = ExtensionManager(shell=self, config=self.config)
2007 2007
2008 2008 def init_plugin_manager(self):
2009 2009 self.plugin_manager = PluginManager(config=self.config)
2010 2010
2011 2011 #-------------------------------------------------------------------------
2012 2012 # Things related to payloads
2013 2013 #-------------------------------------------------------------------------
2014 2014
2015 2015 def init_payload(self):
2016 2016 self.payload_manager = PayloadManager(config=self.config)
2017 2017
2018 2018 #-------------------------------------------------------------------------
2019 2019 # Things related to the prefilter
2020 2020 #-------------------------------------------------------------------------
2021 2021
2022 2022 def init_prefilter(self):
2023 2023 self.prefilter_manager = PrefilterManager(shell=self, config=self.config)
2024 2024 # Ultimately this will be refactored in the new interpreter code, but
2025 2025 # for now, we should expose the main prefilter method (there's legacy
2026 2026 # code out there that may rely on this).
2027 2027 self.prefilter = self.prefilter_manager.prefilter_lines
2028 2028
2029 2029 def auto_rewrite_input(self, cmd):
2030 2030 """Print to the screen the rewritten form of the user's command.
2031 2031
2032 2032 This shows visual feedback by rewriting input lines that cause
2033 2033 automatic calling to kick in, like::
2034 2034
2035 2035 /f x
2036 2036
2037 2037 into::
2038 2038
2039 2039 ------> f(x)
2040 2040
2041 2041 after the user's input prompt. This helps the user understand that the
2042 2042 input line was transformed automatically by IPython.
2043 2043 """
2044 2044 rw = self.displayhook.prompt1.auto_rewrite() + cmd
2045 2045
2046 2046 try:
2047 2047 # plain ascii works better w/ pyreadline, on some machines, so
2048 2048 # we use it and only print uncolored rewrite if we have unicode
2049 2049 rw = str(rw)
2050 2050 print >> io.stdout, rw
2051 2051 except UnicodeEncodeError:
2052 2052 print "------> " + cmd
2053 2053
2054 2054 #-------------------------------------------------------------------------
2055 2055 # Things related to extracting values/expressions from kernel and user_ns
2056 2056 #-------------------------------------------------------------------------
2057 2057
2058 2058 def _simple_error(self):
2059 2059 etype, value = sys.exc_info()[:2]
2060 2060 return u'[ERROR] {e.__name__}: {v}'.format(e=etype, v=value)
2061 2061
2062 2062 def user_variables(self, names):
2063 2063 """Get a list of variable names from the user's namespace.
2064 2064
2065 2065 Parameters
2066 2066 ----------
2067 2067 names : list of strings
2068 2068 A list of names of variables to be read from the user namespace.
2069 2069
2070 2070 Returns
2071 2071 -------
2072 2072 A dict, keyed by the input names and with the repr() of each value.
2073 2073 """
2074 2074 out = {}
2075 2075 user_ns = self.user_ns
2076 2076 for varname in names:
2077 2077 try:
2078 2078 value = repr(user_ns[varname])
2079 2079 except:
2080 2080 value = self._simple_error()
2081 2081 out[varname] = value
2082 2082 return out
2083 2083
2084 2084 def user_expressions(self, expressions):
2085 2085 """Evaluate a dict of expressions in the user's namespace.
2086 2086
2087 2087 Parameters
2088 2088 ----------
2089 2089 expressions : dict
2090 2090 A dict with string keys and string values. The expression values
2091 2091 should be valid Python expressions, each of which will be evaluated
2092 2092 in the user namespace.
2093 2093
2094 2094 Returns
2095 2095 -------
2096 2096 A dict, keyed like the input expressions dict, with the repr() of each
2097 2097 value.
2098 2098 """
2099 2099 out = {}
2100 2100 user_ns = self.user_ns
2101 2101 global_ns = self.user_global_ns
2102 2102 for key, expr in expressions.iteritems():
2103 2103 try:
2104 2104 value = repr(eval(expr, global_ns, user_ns))
2105 2105 except:
2106 2106 value = self._simple_error()
2107 2107 out[key] = value
2108 2108 return out
2109 2109
2110 2110 #-------------------------------------------------------------------------
2111 2111 # Things related to the running of code
2112 2112 #-------------------------------------------------------------------------
2113 2113
2114 2114 def ex(self, cmd):
2115 2115 """Execute a normal python statement in user namespace."""
2116 2116 with self.builtin_trap:
2117 2117 exec cmd in self.user_global_ns, self.user_ns
2118 2118
2119 2119 def ev(self, expr):
2120 2120 """Evaluate python expression expr in user namespace.
2121 2121
2122 2122 Returns the result of evaluation
2123 2123 """
2124 2124 with self.builtin_trap:
2125 2125 return eval(expr, self.user_global_ns, self.user_ns)
2126 2126
2127 2127 def safe_execfile(self, fname, *where, **kw):
2128 2128 """A safe version of the builtin execfile().
2129 2129
2130 2130 This version will never throw an exception, but instead print
2131 2131 helpful error messages to the screen. This only works on pure
2132 2132 Python files with the .py extension.
2133 2133
2134 2134 Parameters
2135 2135 ----------
2136 2136 fname : string
2137 2137 The name of the file to be executed.
2138 2138 where : tuple
2139 2139 One or two namespaces, passed to execfile() as (globals,locals).
2140 2140 If only one is given, it is passed as both.
2141 2141 exit_ignore : bool (False)
2142 2142 If True, then silence SystemExit for non-zero status (it is always
2143 2143 silenced for zero status, as it is so common).
2144 2144 """
2145 2145 kw.setdefault('exit_ignore', False)
2146 2146
2147 2147 fname = os.path.abspath(os.path.expanduser(fname))
2148 2148 # Make sure we have a .py file
2149 2149 if not fname.endswith('.py'):
2150 2150 warn('File must end with .py to be run using execfile: <%s>' % fname)
2151 2151
2152 2152 # Make sure we can open the file
2153 2153 try:
2154 2154 with open(fname) as thefile:
2155 2155 pass
2156 2156 except:
2157 2157 warn('Could not open file <%s> for safe execution.' % fname)
2158 2158 return
2159 2159
2160 2160 # Find things also in current directory. This is needed to mimic the
2161 2161 # behavior of running a script from the system command line, where
2162 2162 # Python inserts the script's directory into sys.path
2163 2163 dname = os.path.dirname(fname)
2164 2164
2165 2165 if isinstance(fname, unicode):
2166 2166 # execfile uses default encoding instead of filesystem encoding
2167 2167 # so unicode filenames will fail
2168 2168 fname = fname.encode(sys.getfilesystemencoding() or sys.getdefaultencoding())
2169 2169
2170 2170 with prepended_to_syspath(dname):
2171 2171 try:
2172 2172 execfile(fname,*where)
2173 2173 except SystemExit, status:
2174 2174 # If the call was made with 0 or None exit status (sys.exit(0)
2175 2175 # or sys.exit() ), don't bother showing a traceback, as both of
2176 2176 # these are considered normal by the OS:
2177 2177 # > python -c'import sys;sys.exit(0)'; echo $?
2178 2178 # 0
2179 2179 # > python -c'import sys;sys.exit()'; echo $?
2180 2180 # 0
2181 2181 # For other exit status, we show the exception unless
2182 2182 # explicitly silenced, but only in short form.
2183 2183 if status.code not in (0, None) and not kw['exit_ignore']:
2184 2184 self.showtraceback(exception_only=True)
2185 2185 except:
2186 2186 self.showtraceback()
2187 2187
2188 2188 def safe_execfile_ipy(self, fname):
2189 2189 """Like safe_execfile, but for .ipy files with IPython syntax.
2190 2190
2191 2191 Parameters
2192 2192 ----------
2193 2193 fname : str
2194 2194 The name of the file to execute. The filename must have a
2195 2195 .ipy extension.
2196 2196 """
2197 2197 fname = os.path.abspath(os.path.expanduser(fname))
2198 2198
2199 2199 # Make sure we have a .py file
2200 2200 if not fname.endswith('.ipy'):
2201 2201 warn('File must end with .py to be run using execfile: <%s>' % fname)
2202 2202
2203 2203 # Make sure we can open the file
2204 2204 try:
2205 2205 with open(fname) as thefile:
2206 2206 pass
2207 2207 except:
2208 2208 warn('Could not open file <%s> for safe execution.' % fname)
2209 2209 return
2210 2210
2211 2211 # Find things also in current directory. This is needed to mimic the
2212 2212 # behavior of running a script from the system command line, where
2213 2213 # Python inserts the script's directory into sys.path
2214 2214 dname = os.path.dirname(fname)
2215 2215
2216 2216 with prepended_to_syspath(dname):
2217 2217 try:
2218 2218 with open(fname) as thefile:
2219 2219 # self.run_cell currently captures all exceptions
2220 2220 # raised in user code. It would be nice if there were
2221 2221 # versions of runlines, execfile that did raise, so
2222 2222 # we could catch the errors.
2223 2223 self.run_cell(thefile.read(), store_history=False)
2224 2224 except:
2225 2225 self.showtraceback()
2226 2226 warn('Unknown failure executing file: <%s>' % fname)
2227 2227
2228 2228 def run_cell(self, raw_cell, store_history=True):
2229 2229 """Run a complete IPython cell.
2230 2230
2231 2231 Parameters
2232 2232 ----------
2233 2233 raw_cell : str
2234 2234 The code (including IPython code such as %magic functions) to run.
2235 2235 store_history : bool
2236 2236 If True, the raw and translated cell will be stored in IPython's
2237 2237 history. For user code calling back into IPython's machinery, this
2238 2238 should be set to False.
2239 2239 """
2240 2240 if (not raw_cell) or raw_cell.isspace():
2241 2241 return
2242 2242
2243 2243 for line in raw_cell.splitlines():
2244 2244 self.input_splitter.push(line)
2245 2245 cell = self.input_splitter.source_reset()
2246 2246
2247 2247 with self.builtin_trap:
2248 2248 prefilter_failed = False
2249 2249 if len(cell.splitlines()) == 1:
2250 2250 try:
2251 2251 # use prefilter_lines to handle trailing newlines
2252 2252 # restore trailing newline for ast.parse
2253 2253 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
2254 2254 except AliasError as e:
2255 2255 error(e)
2256 2256 prefilter_failed=True
2257 2257 except Exception:
2258 2258 # don't allow prefilter errors to crash IPython
2259 2259 self.showtraceback()
2260 2260 prefilter_failed = True
2261 2261
2262 2262 # Store raw and processed history
2263 2263 if store_history:
2264 2264 self.history_manager.store_inputs(self.execution_count,
2265 2265 cell, raw_cell)
2266 2266
2267 2267 self.logger.log(cell, raw_cell)
2268 2268
2269 2269 if not prefilter_failed:
2270 2270 # don't run if prefilter failed
2271 2271 cell_name = self.compile.cache(cell, self.execution_count)
2272 2272
2273 2273 with self.display_trap:
2274 2274 try:
2275 2275 code_ast = ast.parse(cell, filename=cell_name)
2276 2276 except (OverflowError, SyntaxError, ValueError, TypeError,
2277 2277 MemoryError):
2278 2278 self.showsyntaxerror()
2279 2279 self.execution_count += 1
2280 2280 return None
2281 2281
2282 2282 self.run_ast_nodes(code_ast.body, cell_name,
2283 2283 interactivity="last_expr")
2284 2284
2285 2285 # Execute any registered post-execution functions.
2286 2286 for func, status in self._post_execute.iteritems():
2287 2287 if not status:
2288 2288 continue
2289 2289 try:
2290 2290 func()
2291 2291 except:
2292 2292 self.showtraceback()
2293 2293 # Deactivate failing function
2294 2294 self._post_execute[func] = False
2295 2295
2296 2296 if store_history:
2297 2297 # Write output to the database. Does nothing unless
2298 2298 # history output logging is enabled.
2299 2299 self.history_manager.store_output(self.execution_count)
2300 2300 # Each cell is a *single* input, regardless of how many lines it has
2301 2301 self.execution_count += 1
2302 2302
2303 2303 def run_ast_nodes(self, nodelist, cell_name, interactivity='last_expr'):
2304 2304 """Run a sequence of AST nodes. The execution mode depends on the
2305 2305 interactivity parameter.
2306 2306
2307 2307 Parameters
2308 2308 ----------
2309 2309 nodelist : list
2310 2310 A sequence of AST nodes to run.
2311 2311 cell_name : str
2312 2312 Will be passed to the compiler as the filename of the cell. Typically
2313 2313 the value returned by ip.compile.cache(cell).
2314 2314 interactivity : str
2315 2315 'all', 'last', 'last_expr' or 'none', specifying which nodes should be
2316 2316 run interactively (displaying output from expressions). 'last_expr'
2317 2317 will run the last node interactively only if it is an expression (i.e.
2318 2318 expressions in loops or other blocks are not displayed. Other values
2319 2319 for this parameter will raise a ValueError.
2320 2320 """
2321 2321 if not nodelist:
2322 2322 return
2323 2323
2324 2324 if interactivity == 'last_expr':
2325 2325 if isinstance(nodelist[-1], ast.Expr):
2326 2326 interactivity = "last"
2327 2327 else:
2328 2328 interactivity = "none"
2329 2329
2330 2330 if interactivity == 'none':
2331 2331 to_run_exec, to_run_interactive = nodelist, []
2332 2332 elif interactivity == 'last':
2333 2333 to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
2334 2334 elif interactivity == 'all':
2335 2335 to_run_exec, to_run_interactive = [], nodelist
2336 2336 else:
2337 2337 raise ValueError("Interactivity was %r" % interactivity)
2338 2338
2339 2339 exec_count = self.execution_count
2340 2340
2341 2341 for i, node in enumerate(to_run_exec):
2342 2342 mod = ast.Module([node])
2343 2343 code = self.compile(mod, cell_name, "exec")
2344 2344 if self.run_code(code):
2345 2345 return True
2346 2346
2347 2347 for i, node in enumerate(to_run_interactive):
2348 2348 mod = ast.Interactive([node])
2349 2349 code = self.compile(mod, cell_name, "single")
2350 2350 if self.run_code(code):
2351 2351 return True
2352 2352
2353 2353 return False
2354 2354
2355 2355 def run_code(self, code_obj):
2356 2356 """Execute a code object.
2357 2357
2358 2358 When an exception occurs, self.showtraceback() is called to display a
2359 2359 traceback.
2360 2360
2361 2361 Parameters
2362 2362 ----------
2363 2363 code_obj : code object
2364 2364 A compiled code object, to be executed
2365 2365 post_execute : bool [default: True]
2366 2366 whether to call post_execute hooks after this particular execution.
2367 2367
2368 2368 Returns
2369 2369 -------
2370 2370 False : successful execution.
2371 2371 True : an error occurred.
2372 2372 """
2373 2373
2374 2374 # Set our own excepthook in case the user code tries to call it
2375 2375 # directly, so that the IPython crash handler doesn't get triggered
2376 2376 old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
2377 2377
2378 2378 # we save the original sys.excepthook in the instance, in case config
2379 2379 # code (such as magics) needs access to it.
2380 2380 self.sys_excepthook = old_excepthook
2381 2381 outflag = 1 # happens in more places, so it's easier as default
2382 2382 try:
2383 2383 try:
2384 2384 self.hooks.pre_run_code_hook()
2385 2385 #rprint('Running code', repr(code_obj)) # dbg
2386 2386 exec code_obj in self.user_global_ns, self.user_ns
2387 2387 finally:
2388 2388 # Reset our crash handler in place
2389 2389 sys.excepthook = old_excepthook
2390 2390 except SystemExit:
2391 2391 self.showtraceback(exception_only=True)
2392 2392 warn("To exit: use 'exit', 'quit', or Ctrl-D.", level=1)
2393 2393 except self.custom_exceptions:
2394 2394 etype,value,tb = sys.exc_info()
2395 2395 self.CustomTB(etype,value,tb)
2396 2396 except:
2397 2397 self.showtraceback()
2398 2398 else:
2399 2399 outflag = 0
2400 2400 if softspace(sys.stdout, 0):
2401 2401 print
2402 2402
2403 2403 return outflag
2404 2404
2405 2405 # For backwards compatibility
2406 2406 runcode = run_code
2407 2407
2408 2408 #-------------------------------------------------------------------------
2409 2409 # Things related to GUI support and pylab
2410 2410 #-------------------------------------------------------------------------
2411 2411
2412 2412 def enable_pylab(self, gui=None):
2413 2413 raise NotImplementedError('Implement enable_pylab in a subclass')
2414 2414
2415 2415 #-------------------------------------------------------------------------
2416 2416 # Utilities
2417 2417 #-------------------------------------------------------------------------
2418 2418
2419 2419 def var_expand(self,cmd,depth=0):
2420 2420 """Expand python variables in a string.
2421 2421
2422 2422 The depth argument indicates how many frames above the caller should
2423 2423 be walked to look for the local namespace where to expand variables.
2424 2424
2425 2425 The global namespace for expansion is always the user's interactive
2426 2426 namespace.
2427 2427 """
2428 2428 res = ItplNS(cmd, self.user_ns, # globals
2429 2429 # Skip our own frame in searching for locals:
2430 2430 sys._getframe(depth+1).f_locals # locals
2431 2431 )
2432 2432 return str(res).decode(res.codec)
2433 2433
2434 2434 def mktempfile(self, data=None, prefix='ipython_edit_'):
2435 2435 """Make a new tempfile and return its filename.
2436 2436
2437 2437 This makes a call to tempfile.mktemp, but it registers the created
2438 2438 filename internally so ipython cleans it up at exit time.
2439 2439
2440 2440 Optional inputs:
2441 2441
2442 2442 - data(None): if data is given, it gets written out to the temp file
2443 2443 immediately, and the file is closed again."""
2444 2444
2445 2445 filename = tempfile.mktemp('.py', prefix)
2446 2446 self.tempfiles.append(filename)
2447 2447
2448 2448 if data:
2449 2449 tmp_file = open(filename,'w')
2450 2450 tmp_file.write(data)
2451 2451 tmp_file.close()
2452 2452 return filename
2453 2453
2454 2454 # TODO: This should be removed when Term is refactored.
2455 2455 def write(self,data):
2456 2456 """Write a string to the default output"""
2457 2457 io.stdout.write(data)
2458 2458
2459 2459 # TODO: This should be removed when Term is refactored.
2460 2460 def write_err(self,data):
2461 2461 """Write a string to the default error output"""
2462 2462 io.stderr.write(data)
2463 2463
2464 2464 def ask_yes_no(self,prompt,default=True):
2465 2465 if self.quiet:
2466 2466 return True
2467 2467 return ask_yes_no(prompt,default)
2468 2468
2469 2469 def show_usage(self):
2470 2470 """Show a usage message"""
2471 2471 page.page(IPython.core.usage.interactive_usage)
2472 2472
2473 2473 def find_user_code(self, target, raw=True):
2474 2474 """Get a code string from history, file, or a string or macro.
2475 2475
2476 2476 This is mainly used by magic functions.
2477 2477
2478 2478 Parameters
2479 2479 ----------
2480 2480 target : str
2481 2481 A string specifying code to retrieve. This will be tried respectively
2482 2482 as: ranges of input history (see %history for syntax), a filename, or
2483 2483 an expression evaluating to a string or Macro in the user namespace.
2484 2484 raw : bool
2485 2485 If true (default), retrieve raw history. Has no effect on the other
2486 2486 retrieval mechanisms.
2487 2487
2488 2488 Returns
2489 2489 -------
2490 2490 A string of code.
2491 2491
2492 2492 ValueError is raised if nothing is found, and TypeError if it evaluates
2493 2493 to an object of another type. In each case, .args[0] is a printable
2494 2494 message.
2495 2495 """
2496 2496 code = self.extract_input_lines(target, raw=raw) # Grab history
2497 2497 if code:
2498 2498 return code
2499 2499 if os.path.isfile(target): # Read file
2500 2500 return open(target, "r").read()
2501 2501
2502 2502 try: # User namespace
2503 2503 codeobj = eval(target, self.user_ns)
2504 2504 except Exception:
2505 2505 raise ValueError(("'%s' was not found in history, as a file, nor in"
2506 2506 " the user namespace.") % target)
2507 2507 if isinstance(codeobj, basestring):
2508 2508 return codeobj
2509 2509 elif isinstance(codeobj, Macro):
2510 2510 return codeobj.value
2511 2511
2512 2512 raise TypeError("%s is neither a string nor a macro." % target,
2513 2513 codeobj)
2514 2514
2515 2515 #-------------------------------------------------------------------------
2516 2516 # Things related to IPython exiting
2517 2517 #-------------------------------------------------------------------------
2518 2518 def atexit_operations(self):
2519 2519 """This will be executed at the time of exit.
2520 2520
2521 2521 Cleanup operations and saving of persistent data that is done
2522 2522 unconditionally by IPython should be performed here.
2523 2523
2524 2524 For things that may depend on startup flags or platform specifics (such
2525 2525 as having readline or not), register a separate atexit function in the
2526 2526 code that has the appropriate information, rather than trying to
2527 2527 clutter
2528 2528 """
2529 2529 # Cleanup all tempfiles left around
2530 2530 for tfile in self.tempfiles:
2531 2531 try:
2532 2532 os.unlink(tfile)
2533 2533 except OSError:
2534 2534 pass
2535 2535
2536 2536 # Close the history session (this stores the end time and line count)
2537 2537 self.history_manager.end_session()
2538 2538
2539 2539 # Clear all user namespaces to release all references cleanly.
2540 2540 self.reset(new_session=False)
2541 2541
2542 2542 # Run user hooks
2543 2543 self.hooks.shutdown_hook()
2544 2544
2545 2545 def cleanup(self):
2546 2546 self.restore_sys_module_state()
2547 2547
2548 2548
2549 2549 class InteractiveShellABC(object):
2550 2550 """An abstract base class for InteractiveShell."""
2551 2551 __metaclass__ = abc.ABCMeta
2552 2552
2553 2553 InteractiveShellABC.register(InteractiveShell)
@@ -1,3504 +1,3504 b''
1 1 # encoding: utf-8
2 2 """Magic functions for InteractiveShell.
3 3 """
4 4
5 5 #-----------------------------------------------------------------------------
6 6 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de> and
7 7 # Copyright (C) 2001-2007 Fernando Perez <fperez@colorado.edu>
8 8 # Copyright (C) 2008-2009 The IPython Development Team
9 9
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 import __builtin__
19 19 import __future__
20 20 import bdb
21 21 import inspect
22 22 import os
23 23 import sys
24 24 import shutil
25 25 import re
26 26 import time
27 27 import textwrap
28 28 from cStringIO import StringIO
29 29 from getopt import getopt,GetoptError
30 30 from pprint import pformat
31 31 from xmlrpclib import ServerProxy
32 32
33 33 # cProfile was added in Python2.5
34 34 try:
35 35 import cProfile as profile
36 36 import pstats
37 37 except ImportError:
38 38 # profile isn't bundled by default in Debian for license reasons
39 39 try:
40 40 import profile,pstats
41 41 except ImportError:
42 42 profile = pstats = None
43 43
44 44 import IPython
45 45 from IPython.core import debugger, oinspect
46 46 from IPython.core.error import TryNext
47 47 from IPython.core.error import UsageError
48 48 from IPython.core.fakemodule import FakeModule
49 from IPython.core.newapplication import ProfileDir
49 from IPython.core.application import ProfileDir
50 50 from IPython.core.macro import Macro
51 51 from IPython.core import page
52 52 from IPython.core.prefilter import ESC_MAGIC
53 53 from IPython.lib.pylabtools import mpl_runner
54 54 from IPython.external.Itpl import itpl, printpl
55 55 from IPython.testing.skipdoctest import skip_doctest
56 56 from IPython.utils.io import file_read, nlprint
57 57 from IPython.utils.path import get_py_filename
58 58 from IPython.utils.process import arg_split, abbrev_cwd
59 59 from IPython.utils.terminal import set_term_title
60 60 from IPython.utils.text import LSString, SList, format_screen
61 61 from IPython.utils.timing import clock, clock2
62 62 from IPython.utils.warn import warn, error
63 63 from IPython.utils.ipstruct import Struct
64 64 import IPython.utils.generics
65 65
66 66 #-----------------------------------------------------------------------------
67 67 # Utility functions
68 68 #-----------------------------------------------------------------------------
69 69
70 70 def on_off(tag):
71 71 """Return an ON/OFF string for a 1/0 input. Simple utility function."""
72 72 return ['OFF','ON'][tag]
73 73
74 74 class Bunch: pass
75 75
76 76 def compress_dhist(dh):
77 77 head, tail = dh[:-10], dh[-10:]
78 78
79 79 newhead = []
80 80 done = set()
81 81 for h in head:
82 82 if h in done:
83 83 continue
84 84 newhead.append(h)
85 85 done.add(h)
86 86
87 87 return newhead + tail
88 88
89 89 def needs_local_scope(func):
90 90 """Decorator to mark magic functions which need to local scope to run."""
91 91 func.needs_local_scope = True
92 92 return func
93 93
94 94 # Used for exception handling in magic_edit
95 95 class MacroToEdit(ValueError): pass
96 96
97 97 #***************************************************************************
98 98 # Main class implementing Magic functionality
99 99
100 100 # XXX - for some odd reason, if Magic is made a new-style class, we get errors
101 101 # on construction of the main InteractiveShell object. Something odd is going
102 102 # on with super() calls, Configurable and the MRO... For now leave it as-is, but
103 103 # eventually this needs to be clarified.
104 104 # BG: This is because InteractiveShell inherits from this, but is itself a
105 105 # Configurable. This messes up the MRO in some way. The fix is that we need to
106 106 # make Magic a configurable that InteractiveShell does not subclass.
107 107
108 108 class Magic:
109 109 """Magic functions for InteractiveShell.
110 110
111 111 Shell functions which can be reached as %function_name. All magic
112 112 functions should accept a string, which they can parse for their own
113 113 needs. This can make some functions easier to type, eg `%cd ../`
114 114 vs. `%cd("../")`
115 115
116 116 ALL definitions MUST begin with the prefix magic_. The user won't need it
117 117 at the command line, but it is is needed in the definition. """
118 118
119 119 # class globals
120 120 auto_status = ['Automagic is OFF, % prefix IS needed for magic functions.',
121 121 'Automagic is ON, % prefix NOT needed for magic functions.']
122 122
123 123 #......................................................................
124 124 # some utility functions
125 125
126 126 def __init__(self,shell):
127 127
128 128 self.options_table = {}
129 129 if profile is None:
130 130 self.magic_prun = self.profile_missing_notice
131 131 self.shell = shell
132 132
133 133 # namespace for holding state we may need
134 134 self._magic_state = Bunch()
135 135
136 136 def profile_missing_notice(self, *args, **kwargs):
137 137 error("""\
138 138 The profile module could not be found. It has been removed from the standard
139 139 python packages because of its non-free license. To use profiling, install the
140 140 python-profiler package from non-free.""")
141 141
142 142 def default_option(self,fn,optstr):
143 143 """Make an entry in the options_table for fn, with value optstr"""
144 144
145 145 if fn not in self.lsmagic():
146 146 error("%s is not a magic function" % fn)
147 147 self.options_table[fn] = optstr
148 148
149 149 def lsmagic(self):
150 150 """Return a list of currently available magic functions.
151 151
152 152 Gives a list of the bare names after mangling (['ls','cd', ...], not
153 153 ['magic_ls','magic_cd',...]"""
154 154
155 155 # FIXME. This needs a cleanup, in the way the magics list is built.
156 156
157 157 # magics in class definition
158 158 class_magic = lambda fn: fn.startswith('magic_') and \
159 159 callable(Magic.__dict__[fn])
160 160 # in instance namespace (run-time user additions)
161 161 inst_magic = lambda fn: fn.startswith('magic_') and \
162 162 callable(self.__dict__[fn])
163 163 # and bound magics by user (so they can access self):
164 164 inst_bound_magic = lambda fn: fn.startswith('magic_') and \
165 165 callable(self.__class__.__dict__[fn])
166 166 magics = filter(class_magic,Magic.__dict__.keys()) + \
167 167 filter(inst_magic,self.__dict__.keys()) + \
168 168 filter(inst_bound_magic,self.__class__.__dict__.keys())
169 169 out = []
170 170 for fn in set(magics):
171 171 out.append(fn.replace('magic_','',1))
172 172 out.sort()
173 173 return out
174 174
175 175 def extract_input_lines(self, range_str, raw=False):
176 176 """Return as a string a set of input history slices.
177 177
178 178 Inputs:
179 179
180 180 - range_str: the set of slices is given as a string, like
181 181 "~5/6-~4/2 4:8 9", since this function is for use by magic functions
182 182 which get their arguments as strings. The number before the / is the
183 183 session number: ~n goes n back from the current session.
184 184
185 185 Optional inputs:
186 186
187 187 - raw(False): by default, the processed input is used. If this is
188 188 true, the raw input history is used instead.
189 189
190 190 Note that slices can be called with two notations:
191 191
192 192 N:M -> standard python form, means including items N...(M-1).
193 193
194 194 N-M -> include items N..M (closed endpoint)."""
195 195 lines = self.shell.history_manager.\
196 196 get_range_by_str(range_str, raw=raw)
197 197 return "\n".join(x for _, _, x in lines)
198 198
199 199 def arg_err(self,func):
200 200 """Print docstring if incorrect arguments were passed"""
201 201 print 'Error in arguments:'
202 202 print oinspect.getdoc(func)
203 203
204 204 def format_latex(self,strng):
205 205 """Format a string for latex inclusion."""
206 206
207 207 # Characters that need to be escaped for latex:
208 208 escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
209 209 # Magic command names as headers:
210 210 cmd_name_re = re.compile(r'^(%s.*?):' % ESC_MAGIC,
211 211 re.MULTILINE)
212 212 # Magic commands
213 213 cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % ESC_MAGIC,
214 214 re.MULTILINE)
215 215 # Paragraph continue
216 216 par_re = re.compile(r'\\$',re.MULTILINE)
217 217
218 218 # The "\n" symbol
219 219 newline_re = re.compile(r'\\n')
220 220
221 221 # Now build the string for output:
222 222 #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
223 223 strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
224 224 strng)
225 225 strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
226 226 strng = par_re.sub(r'\\\\',strng)
227 227 strng = escape_re.sub(r'\\\1',strng)
228 228 strng = newline_re.sub(r'\\textbackslash{}n',strng)
229 229 return strng
230 230
231 231 def parse_options(self,arg_str,opt_str,*long_opts,**kw):
232 232 """Parse options passed to an argument string.
233 233
234 234 The interface is similar to that of getopt(), but it returns back a
235 235 Struct with the options as keys and the stripped argument string still
236 236 as a string.
237 237
238 238 arg_str is quoted as a true sys.argv vector by using shlex.split.
239 239 This allows us to easily expand variables, glob files, quote
240 240 arguments, etc.
241 241
242 242 Options:
243 243 -mode: default 'string'. If given as 'list', the argument string is
244 244 returned as a list (split on whitespace) instead of a string.
245 245
246 246 -list_all: put all option values in lists. Normally only options
247 247 appearing more than once are put in a list.
248 248
249 249 -posix (True): whether to split the input line in POSIX mode or not,
250 250 as per the conventions outlined in the shlex module from the
251 251 standard library."""
252 252
253 253 # inject default options at the beginning of the input line
254 254 caller = sys._getframe(1).f_code.co_name.replace('magic_','')
255 255 arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
256 256
257 257 mode = kw.get('mode','string')
258 258 if mode not in ['string','list']:
259 259 raise ValueError,'incorrect mode given: %s' % mode
260 260 # Get options
261 261 list_all = kw.get('list_all',0)
262 262 posix = kw.get('posix', os.name == 'posix')
263 263
264 264 # Check if we have more than one argument to warrant extra processing:
265 265 odict = {} # Dictionary with options
266 266 args = arg_str.split()
267 267 if len(args) >= 1:
268 268 # If the list of inputs only has 0 or 1 thing in it, there's no
269 269 # need to look for options
270 270 argv = arg_split(arg_str,posix)
271 271 # Do regular option processing
272 272 try:
273 273 opts,args = getopt(argv,opt_str,*long_opts)
274 274 except GetoptError,e:
275 275 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str,
276 276 " ".join(long_opts)))
277 277 for o,a in opts:
278 278 if o.startswith('--'):
279 279 o = o[2:]
280 280 else:
281 281 o = o[1:]
282 282 try:
283 283 odict[o].append(a)
284 284 except AttributeError:
285 285 odict[o] = [odict[o],a]
286 286 except KeyError:
287 287 if list_all:
288 288 odict[o] = [a]
289 289 else:
290 290 odict[o] = a
291 291
292 292 # Prepare opts,args for return
293 293 opts = Struct(odict)
294 294 if mode == 'string':
295 295 args = ' '.join(args)
296 296
297 297 return opts,args
298 298
299 299 #......................................................................
300 300 # And now the actual magic functions
301 301
302 302 # Functions for IPython shell work (vars,funcs, config, etc)
303 303 def magic_lsmagic(self, parameter_s = ''):
304 304 """List currently available magic functions."""
305 305 mesc = ESC_MAGIC
306 306 print 'Available magic functions:\n'+mesc+\
307 307 (' '+mesc).join(self.lsmagic())
308 308 print '\n' + Magic.auto_status[self.shell.automagic]
309 309 return None
310 310
311 311 def magic_magic(self, parameter_s = ''):
312 312 """Print information about the magic function system.
313 313
314 314 Supported formats: -latex, -brief, -rest
315 315 """
316 316
317 317 mode = ''
318 318 try:
319 319 if parameter_s.split()[0] == '-latex':
320 320 mode = 'latex'
321 321 if parameter_s.split()[0] == '-brief':
322 322 mode = 'brief'
323 323 if parameter_s.split()[0] == '-rest':
324 324 mode = 'rest'
325 325 rest_docs = []
326 326 except:
327 327 pass
328 328
329 329 magic_docs = []
330 330 for fname in self.lsmagic():
331 331 mname = 'magic_' + fname
332 332 for space in (Magic,self,self.__class__):
333 333 try:
334 334 fn = space.__dict__[mname]
335 335 except KeyError:
336 336 pass
337 337 else:
338 338 break
339 339 if mode == 'brief':
340 340 # only first line
341 341 if fn.__doc__:
342 342 fndoc = fn.__doc__.split('\n',1)[0]
343 343 else:
344 344 fndoc = 'No documentation'
345 345 else:
346 346 if fn.__doc__:
347 347 fndoc = fn.__doc__.rstrip()
348 348 else:
349 349 fndoc = 'No documentation'
350 350
351 351
352 352 if mode == 'rest':
353 353 rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(ESC_MAGIC,
354 354 fname,fndoc))
355 355
356 356 else:
357 357 magic_docs.append('%s%s:\n\t%s\n' %(ESC_MAGIC,
358 358 fname,fndoc))
359 359
360 360 magic_docs = ''.join(magic_docs)
361 361
362 362 if mode == 'rest':
363 363 return "".join(rest_docs)
364 364
365 365 if mode == 'latex':
366 366 print self.format_latex(magic_docs)
367 367 return
368 368 else:
369 369 magic_docs = format_screen(magic_docs)
370 370 if mode == 'brief':
371 371 return magic_docs
372 372
373 373 outmsg = """
374 374 IPython's 'magic' functions
375 375 ===========================
376 376
377 377 The magic function system provides a series of functions which allow you to
378 378 control the behavior of IPython itself, plus a lot of system-type
379 379 features. All these functions are prefixed with a % character, but parameters
380 380 are given without parentheses or quotes.
381 381
382 382 NOTE: If you have 'automagic' enabled (via the command line option or with the
383 383 %automagic function), you don't need to type in the % explicitly. By default,
384 384 IPython ships with automagic on, so you should only rarely need the % escape.
385 385
386 386 Example: typing '%cd mydir' (without the quotes) changes you working directory
387 387 to 'mydir', if it exists.
388 388
389 389 You can define your own magic functions to extend the system. See the supplied
390 390 ipythonrc and example-magic.py files for details (in your ipython
391 391 configuration directory, typically $HOME/.config/ipython on Linux or $HOME/.ipython elsewhere).
392 392
393 393 You can also define your own aliased names for magic functions. In your
394 394 ipythonrc file, placing a line like:
395 395
396 396 execute __IPYTHON__.magic_pf = __IPYTHON__.magic_profile
397 397
398 398 will define %pf as a new name for %profile.
399 399
400 400 You can also call magics in code using the magic() function, which IPython
401 401 automatically adds to the builtin namespace. Type 'magic?' for details.
402 402
403 403 For a list of the available magic functions, use %lsmagic. For a description
404 404 of any of them, type %magic_name?, e.g. '%cd?'.
405 405
406 406 Currently the magic system has the following functions:\n"""
407 407
408 408 mesc = ESC_MAGIC
409 409 outmsg = ("%s\n%s\n\nSummary of magic functions (from %slsmagic):"
410 410 "\n\n%s%s\n\n%s" % (outmsg,
411 411 magic_docs,mesc,mesc,
412 412 (' '+mesc).join(self.lsmagic()),
413 413 Magic.auto_status[self.shell.automagic] ) )
414 414 page.page(outmsg)
415 415
416 416 def magic_automagic(self, parameter_s = ''):
417 417 """Make magic functions callable without having to type the initial %.
418 418
419 419 Without argumentsl toggles on/off (when off, you must call it as
420 420 %automagic, of course). With arguments it sets the value, and you can
421 421 use any of (case insensitive):
422 422
423 423 - on,1,True: to activate
424 424
425 425 - off,0,False: to deactivate.
426 426
427 427 Note that magic functions have lowest priority, so if there's a
428 428 variable whose name collides with that of a magic fn, automagic won't
429 429 work for that function (you get the variable instead). However, if you
430 430 delete the variable (del var), the previously shadowed magic function
431 431 becomes visible to automagic again."""
432 432
433 433 arg = parameter_s.lower()
434 434 if parameter_s in ('on','1','true'):
435 435 self.shell.automagic = True
436 436 elif parameter_s in ('off','0','false'):
437 437 self.shell.automagic = False
438 438 else:
439 439 self.shell.automagic = not self.shell.automagic
440 440 print '\n' + Magic.auto_status[self.shell.automagic]
441 441
442 442 @skip_doctest
443 443 def magic_autocall(self, parameter_s = ''):
444 444 """Make functions callable without having to type parentheses.
445 445
446 446 Usage:
447 447
448 448 %autocall [mode]
449 449
450 450 The mode can be one of: 0->Off, 1->Smart, 2->Full. If not given, the
451 451 value is toggled on and off (remembering the previous state).
452 452
453 453 In more detail, these values mean:
454 454
455 455 0 -> fully disabled
456 456
457 457 1 -> active, but do not apply if there are no arguments on the line.
458 458
459 459 In this mode, you get:
460 460
461 461 In [1]: callable
462 462 Out[1]: <built-in function callable>
463 463
464 464 In [2]: callable 'hello'
465 465 ------> callable('hello')
466 466 Out[2]: False
467 467
468 468 2 -> Active always. Even if no arguments are present, the callable
469 469 object is called:
470 470
471 471 In [2]: float
472 472 ------> float()
473 473 Out[2]: 0.0
474 474
475 475 Note that even with autocall off, you can still use '/' at the start of
476 476 a line to treat the first argument on the command line as a function
477 477 and add parentheses to it:
478 478
479 479 In [8]: /str 43
480 480 ------> str(43)
481 481 Out[8]: '43'
482 482
483 483 # all-random (note for auto-testing)
484 484 """
485 485
486 486 if parameter_s:
487 487 arg = int(parameter_s)
488 488 else:
489 489 arg = 'toggle'
490 490
491 491 if not arg in (0,1,2,'toggle'):
492 492 error('Valid modes: (0->Off, 1->Smart, 2->Full')
493 493 return
494 494
495 495 if arg in (0,1,2):
496 496 self.shell.autocall = arg
497 497 else: # toggle
498 498 if self.shell.autocall:
499 499 self._magic_state.autocall_save = self.shell.autocall
500 500 self.shell.autocall = 0
501 501 else:
502 502 try:
503 503 self.shell.autocall = self._magic_state.autocall_save
504 504 except AttributeError:
505 505 self.shell.autocall = self._magic_state.autocall_save = 1
506 506
507 507 print "Automatic calling is:",['OFF','Smart','Full'][self.shell.autocall]
508 508
509 509
510 510 def magic_page(self, parameter_s=''):
511 511 """Pretty print the object and display it through a pager.
512 512
513 513 %page [options] OBJECT
514 514
515 515 If no object is given, use _ (last output).
516 516
517 517 Options:
518 518
519 519 -r: page str(object), don't pretty-print it."""
520 520
521 521 # After a function contributed by Olivier Aubert, slightly modified.
522 522
523 523 # Process options/args
524 524 opts,args = self.parse_options(parameter_s,'r')
525 525 raw = 'r' in opts
526 526
527 527 oname = args and args or '_'
528 528 info = self._ofind(oname)
529 529 if info['found']:
530 530 txt = (raw and str or pformat)( info['obj'] )
531 531 page.page(txt)
532 532 else:
533 533 print 'Object `%s` not found' % oname
534 534
535 535 def magic_profile(self, parameter_s=''):
536 536 """Print your currently active IPython profile."""
537 537 print self.shell.profile
538 538
539 539 def magic_pinfo(self, parameter_s='', namespaces=None):
540 540 """Provide detailed information about an object.
541 541
542 542 '%pinfo object' is just a synonym for object? or ?object."""
543 543
544 544 #print 'pinfo par: <%s>' % parameter_s # dbg
545 545
546 546
547 547 # detail_level: 0 -> obj? , 1 -> obj??
548 548 detail_level = 0
549 549 # We need to detect if we got called as 'pinfo pinfo foo', which can
550 550 # happen if the user types 'pinfo foo?' at the cmd line.
551 551 pinfo,qmark1,oname,qmark2 = \
552 552 re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
553 553 if pinfo or qmark1 or qmark2:
554 554 detail_level = 1
555 555 if "*" in oname:
556 556 self.magic_psearch(oname)
557 557 else:
558 558 self.shell._inspect('pinfo', oname, detail_level=detail_level,
559 559 namespaces=namespaces)
560 560
561 561 def magic_pinfo2(self, parameter_s='', namespaces=None):
562 562 """Provide extra detailed information about an object.
563 563
564 564 '%pinfo2 object' is just a synonym for object?? or ??object."""
565 565 self.shell._inspect('pinfo', parameter_s, detail_level=1,
566 566 namespaces=namespaces)
567 567
568 568 @skip_doctest
569 569 def magic_pdef(self, parameter_s='', namespaces=None):
570 570 """Print the definition header for any callable object.
571 571
572 572 If the object is a class, print the constructor information.
573 573
574 574 Examples
575 575 --------
576 576 ::
577 577
578 578 In [3]: %pdef urllib.urlopen
579 579 urllib.urlopen(url, data=None, proxies=None)
580 580 """
581 581 self._inspect('pdef',parameter_s, namespaces)
582 582
583 583 def magic_pdoc(self, parameter_s='', namespaces=None):
584 584 """Print the docstring for an object.
585 585
586 586 If the given object is a class, it will print both the class and the
587 587 constructor docstrings."""
588 588 self._inspect('pdoc',parameter_s, namespaces)
589 589
590 590 def magic_psource(self, parameter_s='', namespaces=None):
591 591 """Print (or run through pager) the source code for an object."""
592 592 self._inspect('psource',parameter_s, namespaces)
593 593
594 594 def magic_pfile(self, parameter_s=''):
595 595 """Print (or run through pager) the file where an object is defined.
596 596
597 597 The file opens at the line where the object definition begins. IPython
598 598 will honor the environment variable PAGER if set, and otherwise will
599 599 do its best to print the file in a convenient form.
600 600
601 601 If the given argument is not an object currently defined, IPython will
602 602 try to interpret it as a filename (automatically adding a .py extension
603 603 if needed). You can thus use %pfile as a syntax highlighting code
604 604 viewer."""
605 605
606 606 # first interpret argument as an object name
607 607 out = self._inspect('pfile',parameter_s)
608 608 # if not, try the input as a filename
609 609 if out == 'not found':
610 610 try:
611 611 filename = get_py_filename(parameter_s)
612 612 except IOError,msg:
613 613 print msg
614 614 return
615 615 page.page(self.shell.inspector.format(file(filename).read()))
616 616
617 617 def magic_psearch(self, parameter_s=''):
618 618 """Search for object in namespaces by wildcard.
619 619
620 620 %psearch [options] PATTERN [OBJECT TYPE]
621 621
622 622 Note: ? can be used as a synonym for %psearch, at the beginning or at
623 623 the end: both a*? and ?a* are equivalent to '%psearch a*'. Still, the
624 624 rest of the command line must be unchanged (options come first), so
625 625 for example the following forms are equivalent
626 626
627 627 %psearch -i a* function
628 628 -i a* function?
629 629 ?-i a* function
630 630
631 631 Arguments:
632 632
633 633 PATTERN
634 634
635 635 where PATTERN is a string containing * as a wildcard similar to its
636 636 use in a shell. The pattern is matched in all namespaces on the
637 637 search path. By default objects starting with a single _ are not
638 638 matched, many IPython generated objects have a single
639 639 underscore. The default is case insensitive matching. Matching is
640 640 also done on the attributes of objects and not only on the objects
641 641 in a module.
642 642
643 643 [OBJECT TYPE]
644 644
645 645 Is the name of a python type from the types module. The name is
646 646 given in lowercase without the ending type, ex. StringType is
647 647 written string. By adding a type here only objects matching the
648 648 given type are matched. Using all here makes the pattern match all
649 649 types (this is the default).
650 650
651 651 Options:
652 652
653 653 -a: makes the pattern match even objects whose names start with a
654 654 single underscore. These names are normally ommitted from the
655 655 search.
656 656
657 657 -i/-c: make the pattern case insensitive/sensitive. If neither of
658 658 these options is given, the default is read from your ipythonrc
659 659 file. The option name which sets this value is
660 660 'wildcards_case_sensitive'. If this option is not specified in your
661 661 ipythonrc file, IPython's internal default is to do a case sensitive
662 662 search.
663 663
664 664 -e/-s NAMESPACE: exclude/search a given namespace. The pattern you
665 665 specifiy can be searched in any of the following namespaces:
666 666 'builtin', 'user', 'user_global','internal', 'alias', where
667 667 'builtin' and 'user' are the search defaults. Note that you should
668 668 not use quotes when specifying namespaces.
669 669
670 670 'Builtin' contains the python module builtin, 'user' contains all
671 671 user data, 'alias' only contain the shell aliases and no python
672 672 objects, 'internal' contains objects used by IPython. The
673 673 'user_global' namespace is only used by embedded IPython instances,
674 674 and it contains module-level globals. You can add namespaces to the
675 675 search with -s or exclude them with -e (these options can be given
676 676 more than once).
677 677
678 678 Examples:
679 679
680 680 %psearch a* -> objects beginning with an a
681 681 %psearch -e builtin a* -> objects NOT in the builtin space starting in a
682 682 %psearch a* function -> all functions beginning with an a
683 683 %psearch re.e* -> objects beginning with an e in module re
684 684 %psearch r*.e* -> objects that start with e in modules starting in r
685 685 %psearch r*.* string -> all strings in modules beginning with r
686 686
687 687 Case sensitve search:
688 688
689 689 %psearch -c a* list all object beginning with lower case a
690 690
691 691 Show objects beginning with a single _:
692 692
693 693 %psearch -a _* list objects beginning with a single underscore"""
694 694 try:
695 695 parameter_s = parameter_s.encode('ascii')
696 696 except UnicodeEncodeError:
697 697 print 'Python identifiers can only contain ascii characters.'
698 698 return
699 699
700 700 # default namespaces to be searched
701 701 def_search = ['user','builtin']
702 702
703 703 # Process options/args
704 704 opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True)
705 705 opt = opts.get
706 706 shell = self.shell
707 707 psearch = shell.inspector.psearch
708 708
709 709 # select case options
710 710 if opts.has_key('i'):
711 711 ignore_case = True
712 712 elif opts.has_key('c'):
713 713 ignore_case = False
714 714 else:
715 715 ignore_case = not shell.wildcards_case_sensitive
716 716
717 717 # Build list of namespaces to search from user options
718 718 def_search.extend(opt('s',[]))
719 719 ns_exclude = ns_exclude=opt('e',[])
720 720 ns_search = [nm for nm in def_search if nm not in ns_exclude]
721 721
722 722 # Call the actual search
723 723 try:
724 724 psearch(args,shell.ns_table,ns_search,
725 725 show_all=opt('a'),ignore_case=ignore_case)
726 726 except:
727 727 shell.showtraceback()
728 728
729 729 @skip_doctest
730 730 def magic_who_ls(self, parameter_s=''):
731 731 """Return a sorted list of all interactive variables.
732 732
733 733 If arguments are given, only variables of types matching these
734 734 arguments are returned.
735 735
736 736 Examples
737 737 --------
738 738
739 739 Define two variables and list them with who_ls::
740 740
741 741 In [1]: alpha = 123
742 742
743 743 In [2]: beta = 'test'
744 744
745 745 In [3]: %who_ls
746 746 Out[3]: ['alpha', 'beta']
747 747
748 748 In [4]: %who_ls int
749 749 Out[4]: ['alpha']
750 750
751 751 In [5]: %who_ls str
752 752 Out[5]: ['beta']
753 753 """
754 754
755 755 user_ns = self.shell.user_ns
756 756 internal_ns = self.shell.internal_ns
757 757 user_ns_hidden = self.shell.user_ns_hidden
758 758 out = [ i for i in user_ns
759 759 if not i.startswith('_') \
760 760 and not (i in internal_ns or i in user_ns_hidden) ]
761 761
762 762 typelist = parameter_s.split()
763 763 if typelist:
764 764 typeset = set(typelist)
765 765 out = [i for i in out if type(user_ns[i]).__name__ in typeset]
766 766
767 767 out.sort()
768 768 return out
769 769
770 770 @skip_doctest
771 771 def magic_who(self, parameter_s=''):
772 772 """Print all interactive variables, with some minimal formatting.
773 773
774 774 If any arguments are given, only variables whose type matches one of
775 775 these are printed. For example:
776 776
777 777 %who function str
778 778
779 779 will only list functions and strings, excluding all other types of
780 780 variables. To find the proper type names, simply use type(var) at a
781 781 command line to see how python prints type names. For example:
782 782
783 783 In [1]: type('hello')\\
784 784 Out[1]: <type 'str'>
785 785
786 786 indicates that the type name for strings is 'str'.
787 787
788 788 %who always excludes executed names loaded through your configuration
789 789 file and things which are internal to IPython.
790 790
791 791 This is deliberate, as typically you may load many modules and the
792 792 purpose of %who is to show you only what you've manually defined.
793 793
794 794 Examples
795 795 --------
796 796
797 797 Define two variables and list them with who::
798 798
799 799 In [1]: alpha = 123
800 800
801 801 In [2]: beta = 'test'
802 802
803 803 In [3]: %who
804 804 alpha beta
805 805
806 806 In [4]: %who int
807 807 alpha
808 808
809 809 In [5]: %who str
810 810 beta
811 811 """
812 812
813 813 varlist = self.magic_who_ls(parameter_s)
814 814 if not varlist:
815 815 if parameter_s:
816 816 print 'No variables match your requested type.'
817 817 else:
818 818 print 'Interactive namespace is empty.'
819 819 return
820 820
821 821 # if we have variables, move on...
822 822 count = 0
823 823 for i in varlist:
824 824 print i+'\t',
825 825 count += 1
826 826 if count > 8:
827 827 count = 0
828 828 print
829 829 print
830 830
831 831 @skip_doctest
832 832 def magic_whos(self, parameter_s=''):
833 833 """Like %who, but gives some extra information about each variable.
834 834
835 835 The same type filtering of %who can be applied here.
836 836
837 837 For all variables, the type is printed. Additionally it prints:
838 838
839 839 - For {},[],(): their length.
840 840
841 841 - For numpy arrays, a summary with shape, number of
842 842 elements, typecode and size in memory.
843 843
844 844 - Everything else: a string representation, snipping their middle if
845 845 too long.
846 846
847 847 Examples
848 848 --------
849 849
850 850 Define two variables and list them with whos::
851 851
852 852 In [1]: alpha = 123
853 853
854 854 In [2]: beta = 'test'
855 855
856 856 In [3]: %whos
857 857 Variable Type Data/Info
858 858 --------------------------------
859 859 alpha int 123
860 860 beta str test
861 861 """
862 862
863 863 varnames = self.magic_who_ls(parameter_s)
864 864 if not varnames:
865 865 if parameter_s:
866 866 print 'No variables match your requested type.'
867 867 else:
868 868 print 'Interactive namespace is empty.'
869 869 return
870 870
871 871 # if we have variables, move on...
872 872
873 873 # for these types, show len() instead of data:
874 874 seq_types = ['dict', 'list', 'tuple']
875 875
876 876 # for numpy/Numeric arrays, display summary info
877 877 try:
878 878 import numpy
879 879 except ImportError:
880 880 ndarray_type = None
881 881 else:
882 882 ndarray_type = numpy.ndarray.__name__
883 883 try:
884 884 import Numeric
885 885 except ImportError:
886 886 array_type = None
887 887 else:
888 888 array_type = Numeric.ArrayType.__name__
889 889
890 890 # Find all variable names and types so we can figure out column sizes
891 891 def get_vars(i):
892 892 return self.shell.user_ns[i]
893 893
894 894 # some types are well known and can be shorter
895 895 abbrevs = {'IPython.core.macro.Macro' : 'Macro'}
896 896 def type_name(v):
897 897 tn = type(v).__name__
898 898 return abbrevs.get(tn,tn)
899 899
900 900 varlist = map(get_vars,varnames)
901 901
902 902 typelist = []
903 903 for vv in varlist:
904 904 tt = type_name(vv)
905 905
906 906 if tt=='instance':
907 907 typelist.append( abbrevs.get(str(vv.__class__),
908 908 str(vv.__class__)))
909 909 else:
910 910 typelist.append(tt)
911 911
912 912 # column labels and # of spaces as separator
913 913 varlabel = 'Variable'
914 914 typelabel = 'Type'
915 915 datalabel = 'Data/Info'
916 916 colsep = 3
917 917 # variable format strings
918 918 vformat = "$vname.ljust(varwidth)$vtype.ljust(typewidth)"
919 919 vfmt_short = '$vstr[:25]<...>$vstr[-25:]'
920 920 aformat = "%s: %s elems, type `%s`, %s bytes"
921 921 # find the size of the columns to format the output nicely
922 922 varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
923 923 typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
924 924 # table header
925 925 print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
926 926 ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1)
927 927 # and the table itself
928 928 kb = 1024
929 929 Mb = 1048576 # kb**2
930 930 for vname,var,vtype in zip(varnames,varlist,typelist):
931 931 print itpl(vformat),
932 932 if vtype in seq_types:
933 933 print "n="+str(len(var))
934 934 elif vtype in [array_type,ndarray_type]:
935 935 vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
936 936 if vtype==ndarray_type:
937 937 # numpy
938 938 vsize = var.size
939 939 vbytes = vsize*var.itemsize
940 940 vdtype = var.dtype
941 941 else:
942 942 # Numeric
943 943 vsize = Numeric.size(var)
944 944 vbytes = vsize*var.itemsize()
945 945 vdtype = var.typecode()
946 946
947 947 if vbytes < 100000:
948 948 print aformat % (vshape,vsize,vdtype,vbytes)
949 949 else:
950 950 print aformat % (vshape,vsize,vdtype,vbytes),
951 951 if vbytes < Mb:
952 952 print '(%s kb)' % (vbytes/kb,)
953 953 else:
954 954 print '(%s Mb)' % (vbytes/Mb,)
955 955 else:
956 956 try:
957 957 vstr = str(var)
958 958 except UnicodeEncodeError:
959 959 vstr = unicode(var).encode(sys.getdefaultencoding(),
960 960 'backslashreplace')
961 961 vstr = vstr.replace('\n','\\n')
962 962 if len(vstr) < 50:
963 963 print vstr
964 964 else:
965 965 printpl(vfmt_short)
966 966
967 967 def magic_reset(self, parameter_s=''):
968 968 """Resets the namespace by removing all names defined by the user.
969 969
970 970 Parameters
971 971 ----------
972 972 -f : force reset without asking for confirmation.
973 973
974 974 -s : 'Soft' reset: Only clears your namespace, leaving history intact.
975 975 References to objects may be kept. By default (without this option),
976 976 we do a 'hard' reset, giving you a new session and removing all
977 977 references to objects from the current session.
978 978
979 979 Examples
980 980 --------
981 981 In [6]: a = 1
982 982
983 983 In [7]: a
984 984 Out[7]: 1
985 985
986 986 In [8]: 'a' in _ip.user_ns
987 987 Out[8]: True
988 988
989 989 In [9]: %reset -f
990 990
991 991 In [1]: 'a' in _ip.user_ns
992 992 Out[1]: False
993 993 """
994 994 opts, args = self.parse_options(parameter_s,'sf')
995 995 if 'f' in opts:
996 996 ans = True
997 997 else:
998 998 ans = self.shell.ask_yes_no(
999 999 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ")
1000 1000 if not ans:
1001 1001 print 'Nothing done.'
1002 1002 return
1003 1003
1004 1004 if 's' in opts: # Soft reset
1005 1005 user_ns = self.shell.user_ns
1006 1006 for i in self.magic_who_ls():
1007 1007 del(user_ns[i])
1008 1008
1009 1009 else: # Hard reset
1010 1010 self.shell.reset(new_session = False)
1011 1011
1012 1012
1013 1013
1014 1014 def magic_reset_selective(self, parameter_s=''):
1015 1015 """Resets the namespace by removing names defined by the user.
1016 1016
1017 1017 Input/Output history are left around in case you need them.
1018 1018
1019 1019 %reset_selective [-f] regex
1020 1020
1021 1021 No action is taken if regex is not included
1022 1022
1023 1023 Options
1024 1024 -f : force reset without asking for confirmation.
1025 1025
1026 1026 Examples
1027 1027 --------
1028 1028
1029 1029 We first fully reset the namespace so your output looks identical to
1030 1030 this example for pedagogical reasons; in practice you do not need a
1031 1031 full reset.
1032 1032
1033 1033 In [1]: %reset -f
1034 1034
1035 1035 Now, with a clean namespace we can make a few variables and use
1036 1036 %reset_selective to only delete names that match our regexp:
1037 1037
1038 1038 In [2]: a=1; b=2; c=3; b1m=4; b2m=5; b3m=6; b4m=7; b2s=8
1039 1039
1040 1040 In [3]: who_ls
1041 1041 Out[3]: ['a', 'b', 'b1m', 'b2m', 'b2s', 'b3m', 'b4m', 'c']
1042 1042
1043 1043 In [4]: %reset_selective -f b[2-3]m
1044 1044
1045 1045 In [5]: who_ls
1046 1046 Out[5]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
1047 1047
1048 1048 In [6]: %reset_selective -f d
1049 1049
1050 1050 In [7]: who_ls
1051 1051 Out[7]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
1052 1052
1053 1053 In [8]: %reset_selective -f c
1054 1054
1055 1055 In [9]: who_ls
1056 1056 Out[9]: ['a', 'b', 'b1m', 'b2s', 'b4m']
1057 1057
1058 1058 In [10]: %reset_selective -f b
1059 1059
1060 1060 In [11]: who_ls
1061 1061 Out[11]: ['a']
1062 1062 """
1063 1063
1064 1064 opts, regex = self.parse_options(parameter_s,'f')
1065 1065
1066 1066 if opts.has_key('f'):
1067 1067 ans = True
1068 1068 else:
1069 1069 ans = self.shell.ask_yes_no(
1070 1070 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ")
1071 1071 if not ans:
1072 1072 print 'Nothing done.'
1073 1073 return
1074 1074 user_ns = self.shell.user_ns
1075 1075 if not regex:
1076 1076 print 'No regex pattern specified. Nothing done.'
1077 1077 return
1078 1078 else:
1079 1079 try:
1080 1080 m = re.compile(regex)
1081 1081 except TypeError:
1082 1082 raise TypeError('regex must be a string or compiled pattern')
1083 1083 for i in self.magic_who_ls():
1084 1084 if m.search(i):
1085 1085 del(user_ns[i])
1086 1086
1087 1087 def magic_xdel(self, parameter_s=''):
1088 1088 """Delete a variable, trying to clear it from anywhere that
1089 1089 IPython's machinery has references to it. By default, this uses
1090 1090 the identity of the named object in the user namespace to remove
1091 1091 references held under other names. The object is also removed
1092 1092 from the output history.
1093 1093
1094 1094 Options
1095 1095 -n : Delete the specified name from all namespaces, without
1096 1096 checking their identity.
1097 1097 """
1098 1098 opts, varname = self.parse_options(parameter_s,'n')
1099 1099 try:
1100 1100 self.shell.del_var(varname, ('n' in opts))
1101 1101 except (NameError, ValueError) as e:
1102 1102 print type(e).__name__ +": "+ str(e)
1103 1103
1104 1104 def magic_logstart(self,parameter_s=''):
1105 1105 """Start logging anywhere in a session.
1106 1106
1107 1107 %logstart [-o|-r|-t] [log_name [log_mode]]
1108 1108
1109 1109 If no name is given, it defaults to a file named 'ipython_log.py' in your
1110 1110 current directory, in 'rotate' mode (see below).
1111 1111
1112 1112 '%logstart name' saves to file 'name' in 'backup' mode. It saves your
1113 1113 history up to that point and then continues logging.
1114 1114
1115 1115 %logstart takes a second optional parameter: logging mode. This can be one
1116 1116 of (note that the modes are given unquoted):\\
1117 1117 append: well, that says it.\\
1118 1118 backup: rename (if exists) to name~ and start name.\\
1119 1119 global: single logfile in your home dir, appended to.\\
1120 1120 over : overwrite existing log.\\
1121 1121 rotate: create rotating logs name.1~, name.2~, etc.
1122 1122
1123 1123 Options:
1124 1124
1125 1125 -o: log also IPython's output. In this mode, all commands which
1126 1126 generate an Out[NN] prompt are recorded to the logfile, right after
1127 1127 their corresponding input line. The output lines are always
1128 1128 prepended with a '#[Out]# ' marker, so that the log remains valid
1129 1129 Python code.
1130 1130
1131 1131 Since this marker is always the same, filtering only the output from
1132 1132 a log is very easy, using for example a simple awk call:
1133 1133
1134 1134 awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
1135 1135
1136 1136 -r: log 'raw' input. Normally, IPython's logs contain the processed
1137 1137 input, so that user lines are logged in their final form, converted
1138 1138 into valid Python. For example, %Exit is logged as
1139 1139 '_ip.magic("Exit"). If the -r flag is given, all input is logged
1140 1140 exactly as typed, with no transformations applied.
1141 1141
1142 1142 -t: put timestamps before each input line logged (these are put in
1143 1143 comments)."""
1144 1144
1145 1145 opts,par = self.parse_options(parameter_s,'ort')
1146 1146 log_output = 'o' in opts
1147 1147 log_raw_input = 'r' in opts
1148 1148 timestamp = 't' in opts
1149 1149
1150 1150 logger = self.shell.logger
1151 1151
1152 1152 # if no args are given, the defaults set in the logger constructor by
1153 1153 # ipytohn remain valid
1154 1154 if par:
1155 1155 try:
1156 1156 logfname,logmode = par.split()
1157 1157 except:
1158 1158 logfname = par
1159 1159 logmode = 'backup'
1160 1160 else:
1161 1161 logfname = logger.logfname
1162 1162 logmode = logger.logmode
1163 1163 # put logfname into rc struct as if it had been called on the command
1164 1164 # line, so it ends up saved in the log header Save it in case we need
1165 1165 # to restore it...
1166 1166 old_logfile = self.shell.logfile
1167 1167 if logfname:
1168 1168 logfname = os.path.expanduser(logfname)
1169 1169 self.shell.logfile = logfname
1170 1170
1171 1171 loghead = '# IPython log file\n\n'
1172 1172 try:
1173 1173 started = logger.logstart(logfname,loghead,logmode,
1174 1174 log_output,timestamp,log_raw_input)
1175 1175 except:
1176 1176 self.shell.logfile = old_logfile
1177 1177 warn("Couldn't start log: %s" % sys.exc_info()[1])
1178 1178 else:
1179 1179 # log input history up to this point, optionally interleaving
1180 1180 # output if requested
1181 1181
1182 1182 if timestamp:
1183 1183 # disable timestamping for the previous history, since we've
1184 1184 # lost those already (no time machine here).
1185 1185 logger.timestamp = False
1186 1186
1187 1187 if log_raw_input:
1188 1188 input_hist = self.shell.history_manager.input_hist_raw
1189 1189 else:
1190 1190 input_hist = self.shell.history_manager.input_hist_parsed
1191 1191
1192 1192 if log_output:
1193 1193 log_write = logger.log_write
1194 1194 output_hist = self.shell.history_manager.output_hist
1195 1195 for n in range(1,len(input_hist)-1):
1196 1196 log_write(input_hist[n].rstrip() + '\n')
1197 1197 if n in output_hist:
1198 1198 log_write(repr(output_hist[n]),'output')
1199 1199 else:
1200 1200 logger.log_write('\n'.join(input_hist[1:]))
1201 1201 logger.log_write('\n')
1202 1202 if timestamp:
1203 1203 # re-enable timestamping
1204 1204 logger.timestamp = True
1205 1205
1206 1206 print ('Activating auto-logging. '
1207 1207 'Current session state plus future input saved.')
1208 1208 logger.logstate()
1209 1209
1210 1210 def magic_logstop(self,parameter_s=''):
1211 1211 """Fully stop logging and close log file.
1212 1212
1213 1213 In order to start logging again, a new %logstart call needs to be made,
1214 1214 possibly (though not necessarily) with a new filename, mode and other
1215 1215 options."""
1216 1216 self.logger.logstop()
1217 1217
1218 1218 def magic_logoff(self,parameter_s=''):
1219 1219 """Temporarily stop logging.
1220 1220
1221 1221 You must have previously started logging."""
1222 1222 self.shell.logger.switch_log(0)
1223 1223
1224 1224 def magic_logon(self,parameter_s=''):
1225 1225 """Restart logging.
1226 1226
1227 1227 This function is for restarting logging which you've temporarily
1228 1228 stopped with %logoff. For starting logging for the first time, you
1229 1229 must use the %logstart function, which allows you to specify an
1230 1230 optional log filename."""
1231 1231
1232 1232 self.shell.logger.switch_log(1)
1233 1233
1234 1234 def magic_logstate(self,parameter_s=''):
1235 1235 """Print the status of the logging system."""
1236 1236
1237 1237 self.shell.logger.logstate()
1238 1238
1239 1239 def magic_pdb(self, parameter_s=''):
1240 1240 """Control the automatic calling of the pdb interactive debugger.
1241 1241
1242 1242 Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
1243 1243 argument it works as a toggle.
1244 1244
1245 1245 When an exception is triggered, IPython can optionally call the
1246 1246 interactive pdb debugger after the traceback printout. %pdb toggles
1247 1247 this feature on and off.
1248 1248
1249 1249 The initial state of this feature is set in your ipythonrc
1250 1250 configuration file (the variable is called 'pdb').
1251 1251
1252 1252 If you want to just activate the debugger AFTER an exception has fired,
1253 1253 without having to type '%pdb on' and rerunning your code, you can use
1254 1254 the %debug magic."""
1255 1255
1256 1256 par = parameter_s.strip().lower()
1257 1257
1258 1258 if par:
1259 1259 try:
1260 1260 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
1261 1261 except KeyError:
1262 1262 print ('Incorrect argument. Use on/1, off/0, '
1263 1263 'or nothing for a toggle.')
1264 1264 return
1265 1265 else:
1266 1266 # toggle
1267 1267 new_pdb = not self.shell.call_pdb
1268 1268
1269 1269 # set on the shell
1270 1270 self.shell.call_pdb = new_pdb
1271 1271 print 'Automatic pdb calling has been turned',on_off(new_pdb)
1272 1272
1273 1273 def magic_debug(self, parameter_s=''):
1274 1274 """Activate the interactive debugger in post-mortem mode.
1275 1275
1276 1276 If an exception has just occurred, this lets you inspect its stack
1277 1277 frames interactively. Note that this will always work only on the last
1278 1278 traceback that occurred, so you must call this quickly after an
1279 1279 exception that you wish to inspect has fired, because if another one
1280 1280 occurs, it clobbers the previous one.
1281 1281
1282 1282 If you want IPython to automatically do this on every exception, see
1283 1283 the %pdb magic for more details.
1284 1284 """
1285 1285 self.shell.debugger(force=True)
1286 1286
1287 1287 @skip_doctest
1288 1288 def magic_prun(self, parameter_s ='',user_mode=1,
1289 1289 opts=None,arg_lst=None,prog_ns=None):
1290 1290
1291 1291 """Run a statement through the python code profiler.
1292 1292
1293 1293 Usage:
1294 1294 %prun [options] statement
1295 1295
1296 1296 The given statement (which doesn't require quote marks) is run via the
1297 1297 python profiler in a manner similar to the profile.run() function.
1298 1298 Namespaces are internally managed to work correctly; profile.run
1299 1299 cannot be used in IPython because it makes certain assumptions about
1300 1300 namespaces which do not hold under IPython.
1301 1301
1302 1302 Options:
1303 1303
1304 1304 -l <limit>: you can place restrictions on what or how much of the
1305 1305 profile gets printed. The limit value can be:
1306 1306
1307 1307 * A string: only information for function names containing this string
1308 1308 is printed.
1309 1309
1310 1310 * An integer: only these many lines are printed.
1311 1311
1312 1312 * A float (between 0 and 1): this fraction of the report is printed
1313 1313 (for example, use a limit of 0.4 to see the topmost 40% only).
1314 1314
1315 1315 You can combine several limits with repeated use of the option. For
1316 1316 example, '-l __init__ -l 5' will print only the topmost 5 lines of
1317 1317 information about class constructors.
1318 1318
1319 1319 -r: return the pstats.Stats object generated by the profiling. This
1320 1320 object has all the information about the profile in it, and you can
1321 1321 later use it for further analysis or in other functions.
1322 1322
1323 1323 -s <key>: sort profile by given key. You can provide more than one key
1324 1324 by using the option several times: '-s key1 -s key2 -s key3...'. The
1325 1325 default sorting key is 'time'.
1326 1326
1327 1327 The following is copied verbatim from the profile documentation
1328 1328 referenced below:
1329 1329
1330 1330 When more than one key is provided, additional keys are used as
1331 1331 secondary criteria when the there is equality in all keys selected
1332 1332 before them.
1333 1333
1334 1334 Abbreviations can be used for any key names, as long as the
1335 1335 abbreviation is unambiguous. The following are the keys currently
1336 1336 defined:
1337 1337
1338 1338 Valid Arg Meaning
1339 1339 "calls" call count
1340 1340 "cumulative" cumulative time
1341 1341 "file" file name
1342 1342 "module" file name
1343 1343 "pcalls" primitive call count
1344 1344 "line" line number
1345 1345 "name" function name
1346 1346 "nfl" name/file/line
1347 1347 "stdname" standard name
1348 1348 "time" internal time
1349 1349
1350 1350 Note that all sorts on statistics are in descending order (placing
1351 1351 most time consuming items first), where as name, file, and line number
1352 1352 searches are in ascending order (i.e., alphabetical). The subtle
1353 1353 distinction between "nfl" and "stdname" is that the standard name is a
1354 1354 sort of the name as printed, which means that the embedded line
1355 1355 numbers get compared in an odd way. For example, lines 3, 20, and 40
1356 1356 would (if the file names were the same) appear in the string order
1357 1357 "20" "3" and "40". In contrast, "nfl" does a numeric compare of the
1358 1358 line numbers. In fact, sort_stats("nfl") is the same as
1359 1359 sort_stats("name", "file", "line").
1360 1360
1361 1361 -T <filename>: save profile results as shown on screen to a text
1362 1362 file. The profile is still shown on screen.
1363 1363
1364 1364 -D <filename>: save (via dump_stats) profile statistics to given
1365 1365 filename. This data is in a format understod by the pstats module, and
1366 1366 is generated by a call to the dump_stats() method of profile
1367 1367 objects. The profile is still shown on screen.
1368 1368
1369 1369 If you want to run complete programs under the profiler's control, use
1370 1370 '%run -p [prof_opts] filename.py [args to program]' where prof_opts
1371 1371 contains profiler specific options as described here.
1372 1372
1373 1373 You can read the complete documentation for the profile module with::
1374 1374
1375 1375 In [1]: import profile; profile.help()
1376 1376 """
1377 1377
1378 1378 opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
1379 1379 # protect user quote marks
1380 1380 parameter_s = parameter_s.replace('"',r'\"').replace("'",r"\'")
1381 1381
1382 1382 if user_mode: # regular user call
1383 1383 opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:',
1384 1384 list_all=1)
1385 1385 namespace = self.shell.user_ns
1386 1386 else: # called to run a program by %run -p
1387 1387 try:
1388 1388 filename = get_py_filename(arg_lst[0])
1389 1389 except IOError,msg:
1390 1390 error(msg)
1391 1391 return
1392 1392
1393 1393 arg_str = 'execfile(filename,prog_ns)'
1394 1394 namespace = locals()
1395 1395
1396 1396 opts.merge(opts_def)
1397 1397
1398 1398 prof = profile.Profile()
1399 1399 try:
1400 1400 prof = prof.runctx(arg_str,namespace,namespace)
1401 1401 sys_exit = ''
1402 1402 except SystemExit:
1403 1403 sys_exit = """*** SystemExit exception caught in code being profiled."""
1404 1404
1405 1405 stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
1406 1406
1407 1407 lims = opts.l
1408 1408 if lims:
1409 1409 lims = [] # rebuild lims with ints/floats/strings
1410 1410 for lim in opts.l:
1411 1411 try:
1412 1412 lims.append(int(lim))
1413 1413 except ValueError:
1414 1414 try:
1415 1415 lims.append(float(lim))
1416 1416 except ValueError:
1417 1417 lims.append(lim)
1418 1418
1419 1419 # Trap output.
1420 1420 stdout_trap = StringIO()
1421 1421
1422 1422 if hasattr(stats,'stream'):
1423 1423 # In newer versions of python, the stats object has a 'stream'
1424 1424 # attribute to write into.
1425 1425 stats.stream = stdout_trap
1426 1426 stats.print_stats(*lims)
1427 1427 else:
1428 1428 # For older versions, we manually redirect stdout during printing
1429 1429 sys_stdout = sys.stdout
1430 1430 try:
1431 1431 sys.stdout = stdout_trap
1432 1432 stats.print_stats(*lims)
1433 1433 finally:
1434 1434 sys.stdout = sys_stdout
1435 1435
1436 1436 output = stdout_trap.getvalue()
1437 1437 output = output.rstrip()
1438 1438
1439 1439 page.page(output)
1440 1440 print sys_exit,
1441 1441
1442 1442 dump_file = opts.D[0]
1443 1443 text_file = opts.T[0]
1444 1444 if dump_file:
1445 1445 prof.dump_stats(dump_file)
1446 1446 print '\n*** Profile stats marshalled to file',\
1447 1447 `dump_file`+'.',sys_exit
1448 1448 if text_file:
1449 1449 pfile = file(text_file,'w')
1450 1450 pfile.write(output)
1451 1451 pfile.close()
1452 1452 print '\n*** Profile printout saved to text file',\
1453 1453 `text_file`+'.',sys_exit
1454 1454
1455 1455 if opts.has_key('r'):
1456 1456 return stats
1457 1457 else:
1458 1458 return None
1459 1459
1460 1460 @skip_doctest
1461 1461 def magic_run(self, parameter_s ='',runner=None,
1462 1462 file_finder=get_py_filename):
1463 1463 """Run the named file inside IPython as a program.
1464 1464
1465 1465 Usage:\\
1466 1466 %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
1467 1467
1468 1468 Parameters after the filename are passed as command-line arguments to
1469 1469 the program (put in sys.argv). Then, control returns to IPython's
1470 1470 prompt.
1471 1471
1472 1472 This is similar to running at a system prompt:\\
1473 1473 $ python file args\\
1474 1474 but with the advantage of giving you IPython's tracebacks, and of
1475 1475 loading all variables into your interactive namespace for further use
1476 1476 (unless -p is used, see below).
1477 1477
1478 1478 The file is executed in a namespace initially consisting only of
1479 1479 __name__=='__main__' and sys.argv constructed as indicated. It thus
1480 1480 sees its environment as if it were being run as a stand-alone program
1481 1481 (except for sharing global objects such as previously imported
1482 1482 modules). But after execution, the IPython interactive namespace gets
1483 1483 updated with all variables defined in the program (except for __name__
1484 1484 and sys.argv). This allows for very convenient loading of code for
1485 1485 interactive work, while giving each program a 'clean sheet' to run in.
1486 1486
1487 1487 Options:
1488 1488
1489 1489 -n: __name__ is NOT set to '__main__', but to the running file's name
1490 1490 without extension (as python does under import). This allows running
1491 1491 scripts and reloading the definitions in them without calling code
1492 1492 protected by an ' if __name__ == "__main__" ' clause.
1493 1493
1494 1494 -i: run the file in IPython's namespace instead of an empty one. This
1495 1495 is useful if you are experimenting with code written in a text editor
1496 1496 which depends on variables defined interactively.
1497 1497
1498 1498 -e: ignore sys.exit() calls or SystemExit exceptions in the script
1499 1499 being run. This is particularly useful if IPython is being used to
1500 1500 run unittests, which always exit with a sys.exit() call. In such
1501 1501 cases you are interested in the output of the test results, not in
1502 1502 seeing a traceback of the unittest module.
1503 1503
1504 1504 -t: print timing information at the end of the run. IPython will give
1505 1505 you an estimated CPU time consumption for your script, which under
1506 1506 Unix uses the resource module to avoid the wraparound problems of
1507 1507 time.clock(). Under Unix, an estimate of time spent on system tasks
1508 1508 is also given (for Windows platforms this is reported as 0.0).
1509 1509
1510 1510 If -t is given, an additional -N<N> option can be given, where <N>
1511 1511 must be an integer indicating how many times you want the script to
1512 1512 run. The final timing report will include total and per run results.
1513 1513
1514 1514 For example (testing the script uniq_stable.py):
1515 1515
1516 1516 In [1]: run -t uniq_stable
1517 1517
1518 1518 IPython CPU timings (estimated):\\
1519 1519 User : 0.19597 s.\\
1520 1520 System: 0.0 s.\\
1521 1521
1522 1522 In [2]: run -t -N5 uniq_stable
1523 1523
1524 1524 IPython CPU timings (estimated):\\
1525 1525 Total runs performed: 5\\
1526 1526 Times : Total Per run\\
1527 1527 User : 0.910862 s, 0.1821724 s.\\
1528 1528 System: 0.0 s, 0.0 s.
1529 1529
1530 1530 -d: run your program under the control of pdb, the Python debugger.
1531 1531 This allows you to execute your program step by step, watch variables,
1532 1532 etc. Internally, what IPython does is similar to calling:
1533 1533
1534 1534 pdb.run('execfile("YOURFILENAME")')
1535 1535
1536 1536 with a breakpoint set on line 1 of your file. You can change the line
1537 1537 number for this automatic breakpoint to be <N> by using the -bN option
1538 1538 (where N must be an integer). For example:
1539 1539
1540 1540 %run -d -b40 myscript
1541 1541
1542 1542 will set the first breakpoint at line 40 in myscript.py. Note that
1543 1543 the first breakpoint must be set on a line which actually does
1544 1544 something (not a comment or docstring) for it to stop execution.
1545 1545
1546 1546 When the pdb debugger starts, you will see a (Pdb) prompt. You must
1547 1547 first enter 'c' (without qoutes) to start execution up to the first
1548 1548 breakpoint.
1549 1549
1550 1550 Entering 'help' gives information about the use of the debugger. You
1551 1551 can easily see pdb's full documentation with "import pdb;pdb.help()"
1552 1552 at a prompt.
1553 1553
1554 1554 -p: run program under the control of the Python profiler module (which
1555 1555 prints a detailed report of execution times, function calls, etc).
1556 1556
1557 1557 You can pass other options after -p which affect the behavior of the
1558 1558 profiler itself. See the docs for %prun for details.
1559 1559
1560 1560 In this mode, the program's variables do NOT propagate back to the
1561 1561 IPython interactive namespace (because they remain in the namespace
1562 1562 where the profiler executes them).
1563 1563
1564 1564 Internally this triggers a call to %prun, see its documentation for
1565 1565 details on the options available specifically for profiling.
1566 1566
1567 1567 There is one special usage for which the text above doesn't apply:
1568 1568 if the filename ends with .ipy, the file is run as ipython script,
1569 1569 just as if the commands were written on IPython prompt.
1570 1570 """
1571 1571
1572 1572 # get arguments and set sys.argv for program to be run.
1573 1573 opts,arg_lst = self.parse_options(parameter_s,'nidtN:b:pD:l:rs:T:e',
1574 1574 mode='list',list_all=1)
1575 1575
1576 1576 try:
1577 1577 filename = file_finder(arg_lst[0])
1578 1578 except IndexError:
1579 1579 warn('you must provide at least a filename.')
1580 1580 print '\n%run:\n',oinspect.getdoc(self.magic_run)
1581 1581 return
1582 1582 except IOError,msg:
1583 1583 error(msg)
1584 1584 return
1585 1585
1586 1586 if filename.lower().endswith('.ipy'):
1587 1587 self.shell.safe_execfile_ipy(filename)
1588 1588 return
1589 1589
1590 1590 # Control the response to exit() calls made by the script being run
1591 1591 exit_ignore = opts.has_key('e')
1592 1592
1593 1593 # Make sure that the running script gets a proper sys.argv as if it
1594 1594 # were run from a system shell.
1595 1595 save_argv = sys.argv # save it for later restoring
1596 1596 sys.argv = [filename]+ arg_lst[1:] # put in the proper filename
1597 1597
1598 1598 if opts.has_key('i'):
1599 1599 # Run in user's interactive namespace
1600 1600 prog_ns = self.shell.user_ns
1601 1601 __name__save = self.shell.user_ns['__name__']
1602 1602 prog_ns['__name__'] = '__main__'
1603 1603 main_mod = self.shell.new_main_mod(prog_ns)
1604 1604 else:
1605 1605 # Run in a fresh, empty namespace
1606 1606 if opts.has_key('n'):
1607 1607 name = os.path.splitext(os.path.basename(filename))[0]
1608 1608 else:
1609 1609 name = '__main__'
1610 1610
1611 1611 main_mod = self.shell.new_main_mod()
1612 1612 prog_ns = main_mod.__dict__
1613 1613 prog_ns['__name__'] = name
1614 1614
1615 1615 # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
1616 1616 # set the __file__ global in the script's namespace
1617 1617 prog_ns['__file__'] = filename
1618 1618
1619 1619 # pickle fix. See interactiveshell for an explanation. But we need to make sure
1620 1620 # that, if we overwrite __main__, we replace it at the end
1621 1621 main_mod_name = prog_ns['__name__']
1622 1622
1623 1623 if main_mod_name == '__main__':
1624 1624 restore_main = sys.modules['__main__']
1625 1625 else:
1626 1626 restore_main = False
1627 1627
1628 1628 # This needs to be undone at the end to prevent holding references to
1629 1629 # every single object ever created.
1630 1630 sys.modules[main_mod_name] = main_mod
1631 1631
1632 1632 try:
1633 1633 stats = None
1634 1634 with self.readline_no_record:
1635 1635 if opts.has_key('p'):
1636 1636 stats = self.magic_prun('',0,opts,arg_lst,prog_ns)
1637 1637 else:
1638 1638 if opts.has_key('d'):
1639 1639 deb = debugger.Pdb(self.shell.colors)
1640 1640 # reset Breakpoint state, which is moronically kept
1641 1641 # in a class
1642 1642 bdb.Breakpoint.next = 1
1643 1643 bdb.Breakpoint.bplist = {}
1644 1644 bdb.Breakpoint.bpbynumber = [None]
1645 1645 # Set an initial breakpoint to stop execution
1646 1646 maxtries = 10
1647 1647 bp = int(opts.get('b',[1])[0])
1648 1648 checkline = deb.checkline(filename,bp)
1649 1649 if not checkline:
1650 1650 for bp in range(bp+1,bp+maxtries+1):
1651 1651 if deb.checkline(filename,bp):
1652 1652 break
1653 1653 else:
1654 1654 msg = ("\nI failed to find a valid line to set "
1655 1655 "a breakpoint\n"
1656 1656 "after trying up to line: %s.\n"
1657 1657 "Please set a valid breakpoint manually "
1658 1658 "with the -b option." % bp)
1659 1659 error(msg)
1660 1660 return
1661 1661 # if we find a good linenumber, set the breakpoint
1662 1662 deb.do_break('%s:%s' % (filename,bp))
1663 1663 # Start file run
1664 1664 print "NOTE: Enter 'c' at the",
1665 1665 print "%s prompt to start your script." % deb.prompt
1666 1666 try:
1667 1667 deb.run('execfile("%s")' % filename,prog_ns)
1668 1668
1669 1669 except:
1670 1670 etype, value, tb = sys.exc_info()
1671 1671 # Skip three frames in the traceback: the %run one,
1672 1672 # one inside bdb.py, and the command-line typed by the
1673 1673 # user (run by exec in pdb itself).
1674 1674 self.shell.InteractiveTB(etype,value,tb,tb_offset=3)
1675 1675 else:
1676 1676 if runner is None:
1677 1677 runner = self.shell.safe_execfile
1678 1678 if opts.has_key('t'):
1679 1679 # timed execution
1680 1680 try:
1681 1681 nruns = int(opts['N'][0])
1682 1682 if nruns < 1:
1683 1683 error('Number of runs must be >=1')
1684 1684 return
1685 1685 except (KeyError):
1686 1686 nruns = 1
1687 1687 if nruns == 1:
1688 1688 t0 = clock2()
1689 1689 runner(filename,prog_ns,prog_ns,
1690 1690 exit_ignore=exit_ignore)
1691 1691 t1 = clock2()
1692 1692 t_usr = t1[0]-t0[0]
1693 1693 t_sys = t1[1]-t0[1]
1694 1694 print "\nIPython CPU timings (estimated):"
1695 1695 print " User : %10s s." % t_usr
1696 1696 print " System: %10s s." % t_sys
1697 1697 else:
1698 1698 runs = range(nruns)
1699 1699 t0 = clock2()
1700 1700 for nr in runs:
1701 1701 runner(filename,prog_ns,prog_ns,
1702 1702 exit_ignore=exit_ignore)
1703 1703 t1 = clock2()
1704 1704 t_usr = t1[0]-t0[0]
1705 1705 t_sys = t1[1]-t0[1]
1706 1706 print "\nIPython CPU timings (estimated):"
1707 1707 print "Total runs performed:",nruns
1708 1708 print " Times : %10s %10s" % ('Total','Per run')
1709 1709 print " User : %10s s, %10s s." % (t_usr,t_usr/nruns)
1710 1710 print " System: %10s s, %10s s." % (t_sys,t_sys/nruns)
1711 1711
1712 1712 else:
1713 1713 # regular execution
1714 1714 runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore)
1715 1715
1716 1716 if opts.has_key('i'):
1717 1717 self.shell.user_ns['__name__'] = __name__save
1718 1718 else:
1719 1719 # The shell MUST hold a reference to prog_ns so after %run
1720 1720 # exits, the python deletion mechanism doesn't zero it out
1721 1721 # (leaving dangling references).
1722 1722 self.shell.cache_main_mod(prog_ns,filename)
1723 1723 # update IPython interactive namespace
1724 1724
1725 1725 # Some forms of read errors on the file may mean the
1726 1726 # __name__ key was never set; using pop we don't have to
1727 1727 # worry about a possible KeyError.
1728 1728 prog_ns.pop('__name__', None)
1729 1729
1730 1730 self.shell.user_ns.update(prog_ns)
1731 1731 finally:
1732 1732 # It's a bit of a mystery why, but __builtins__ can change from
1733 1733 # being a module to becoming a dict missing some key data after
1734 1734 # %run. As best I can see, this is NOT something IPython is doing
1735 1735 # at all, and similar problems have been reported before:
1736 1736 # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
1737 1737 # Since this seems to be done by the interpreter itself, the best
1738 1738 # we can do is to at least restore __builtins__ for the user on
1739 1739 # exit.
1740 1740 self.shell.user_ns['__builtins__'] = __builtin__
1741 1741
1742 1742 # Ensure key global structures are restored
1743 1743 sys.argv = save_argv
1744 1744 if restore_main:
1745 1745 sys.modules['__main__'] = restore_main
1746 1746 else:
1747 1747 # Remove from sys.modules the reference to main_mod we'd
1748 1748 # added. Otherwise it will trap references to objects
1749 1749 # contained therein.
1750 1750 del sys.modules[main_mod_name]
1751 1751
1752 1752 return stats
1753 1753
1754 1754 @skip_doctest
1755 1755 def magic_timeit(self, parameter_s =''):
1756 1756 """Time execution of a Python statement or expression
1757 1757
1758 1758 Usage:\\
1759 1759 %timeit [-n<N> -r<R> [-t|-c]] statement
1760 1760
1761 1761 Time execution of a Python statement or expression using the timeit
1762 1762 module.
1763 1763
1764 1764 Options:
1765 1765 -n<N>: execute the given statement <N> times in a loop. If this value
1766 1766 is not given, a fitting value is chosen.
1767 1767
1768 1768 -r<R>: repeat the loop iteration <R> times and take the best result.
1769 1769 Default: 3
1770 1770
1771 1771 -t: use time.time to measure the time, which is the default on Unix.
1772 1772 This function measures wall time.
1773 1773
1774 1774 -c: use time.clock to measure the time, which is the default on
1775 1775 Windows and measures wall time. On Unix, resource.getrusage is used
1776 1776 instead and returns the CPU user time.
1777 1777
1778 1778 -p<P>: use a precision of <P> digits to display the timing result.
1779 1779 Default: 3
1780 1780
1781 1781
1782 1782 Examples:
1783 1783
1784 1784 In [1]: %timeit pass
1785 1785 10000000 loops, best of 3: 53.3 ns per loop
1786 1786
1787 1787 In [2]: u = None
1788 1788
1789 1789 In [3]: %timeit u is None
1790 1790 10000000 loops, best of 3: 184 ns per loop
1791 1791
1792 1792 In [4]: %timeit -r 4 u == None
1793 1793 1000000 loops, best of 4: 242 ns per loop
1794 1794
1795 1795 In [5]: import time
1796 1796
1797 1797 In [6]: %timeit -n1 time.sleep(2)
1798 1798 1 loops, best of 3: 2 s per loop
1799 1799
1800 1800
1801 1801 The times reported by %timeit will be slightly higher than those
1802 1802 reported by the timeit.py script when variables are accessed. This is
1803 1803 due to the fact that %timeit executes the statement in the namespace
1804 1804 of the shell, compared with timeit.py, which uses a single setup
1805 1805 statement to import function or create variables. Generally, the bias
1806 1806 does not matter as long as results from timeit.py are not mixed with
1807 1807 those from %timeit."""
1808 1808
1809 1809 import timeit
1810 1810 import math
1811 1811
1812 1812 # XXX: Unfortunately the unicode 'micro' symbol can cause problems in
1813 1813 # certain terminals. Until we figure out a robust way of
1814 1814 # auto-detecting if the terminal can deal with it, use plain 'us' for
1815 1815 # microseconds. I am really NOT happy about disabling the proper
1816 1816 # 'micro' prefix, but crashing is worse... If anyone knows what the
1817 1817 # right solution for this is, I'm all ears...
1818 1818 #
1819 1819 # Note: using
1820 1820 #
1821 1821 # s = u'\xb5'
1822 1822 # s.encode(sys.getdefaultencoding())
1823 1823 #
1824 1824 # is not sufficient, as I've seen terminals where that fails but
1825 1825 # print s
1826 1826 #
1827 1827 # succeeds
1828 1828 #
1829 1829 # See bug: https://bugs.launchpad.net/ipython/+bug/348466
1830 1830
1831 1831 #units = [u"s", u"ms",u'\xb5',"ns"]
1832 1832 units = [u"s", u"ms",u'us',"ns"]
1833 1833
1834 1834 scaling = [1, 1e3, 1e6, 1e9]
1835 1835
1836 1836 opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
1837 1837 posix=False)
1838 1838 if stmt == "":
1839 1839 return
1840 1840 timefunc = timeit.default_timer
1841 1841 number = int(getattr(opts, "n", 0))
1842 1842 repeat = int(getattr(opts, "r", timeit.default_repeat))
1843 1843 precision = int(getattr(opts, "p", 3))
1844 1844 if hasattr(opts, "t"):
1845 1845 timefunc = time.time
1846 1846 if hasattr(opts, "c"):
1847 1847 timefunc = clock
1848 1848
1849 1849 timer = timeit.Timer(timer=timefunc)
1850 1850 # this code has tight coupling to the inner workings of timeit.Timer,
1851 1851 # but is there a better way to achieve that the code stmt has access
1852 1852 # to the shell namespace?
1853 1853
1854 1854 src = timeit.template % {'stmt': timeit.reindent(stmt, 8),
1855 1855 'setup': "pass"}
1856 1856 # Track compilation time so it can be reported if too long
1857 1857 # Minimum time above which compilation time will be reported
1858 1858 tc_min = 0.1
1859 1859
1860 1860 t0 = clock()
1861 1861 code = compile(src, "<magic-timeit>", "exec")
1862 1862 tc = clock()-t0
1863 1863
1864 1864 ns = {}
1865 1865 exec code in self.shell.user_ns, ns
1866 1866 timer.inner = ns["inner"]
1867 1867
1868 1868 if number == 0:
1869 1869 # determine number so that 0.2 <= total time < 2.0
1870 1870 number = 1
1871 1871 for i in range(1, 10):
1872 1872 if timer.timeit(number) >= 0.2:
1873 1873 break
1874 1874 number *= 10
1875 1875
1876 1876 best = min(timer.repeat(repeat, number)) / number
1877 1877
1878 1878 if best > 0.0 and best < 1000.0:
1879 1879 order = min(-int(math.floor(math.log10(best)) // 3), 3)
1880 1880 elif best >= 1000.0:
1881 1881 order = 0
1882 1882 else:
1883 1883 order = 3
1884 1884 print u"%d loops, best of %d: %.*g %s per loop" % (number, repeat,
1885 1885 precision,
1886 1886 best * scaling[order],
1887 1887 units[order])
1888 1888 if tc > tc_min:
1889 1889 print "Compiler time: %.2f s" % tc
1890 1890
1891 1891 @skip_doctest
1892 1892 @needs_local_scope
1893 1893 def magic_time(self,parameter_s = ''):
1894 1894 """Time execution of a Python statement or expression.
1895 1895
1896 1896 The CPU and wall clock times are printed, and the value of the
1897 1897 expression (if any) is returned. Note that under Win32, system time
1898 1898 is always reported as 0, since it can not be measured.
1899 1899
1900 1900 This function provides very basic timing functionality. In Python
1901 1901 2.3, the timeit module offers more control and sophistication, so this
1902 1902 could be rewritten to use it (patches welcome).
1903 1903
1904 1904 Some examples:
1905 1905
1906 1906 In [1]: time 2**128
1907 1907 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1908 1908 Wall time: 0.00
1909 1909 Out[1]: 340282366920938463463374607431768211456L
1910 1910
1911 1911 In [2]: n = 1000000
1912 1912
1913 1913 In [3]: time sum(range(n))
1914 1914 CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
1915 1915 Wall time: 1.37
1916 1916 Out[3]: 499999500000L
1917 1917
1918 1918 In [4]: time print 'hello world'
1919 1919 hello world
1920 1920 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1921 1921 Wall time: 0.00
1922 1922
1923 1923 Note that the time needed by Python to compile the given expression
1924 1924 will be reported if it is more than 0.1s. In this example, the
1925 1925 actual exponentiation is done by Python at compilation time, so while
1926 1926 the expression can take a noticeable amount of time to compute, that
1927 1927 time is purely due to the compilation:
1928 1928
1929 1929 In [5]: time 3**9999;
1930 1930 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1931 1931 Wall time: 0.00 s
1932 1932
1933 1933 In [6]: time 3**999999;
1934 1934 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1935 1935 Wall time: 0.00 s
1936 1936 Compiler : 0.78 s
1937 1937 """
1938 1938
1939 1939 # fail immediately if the given expression can't be compiled
1940 1940
1941 1941 expr = self.shell.prefilter(parameter_s,False)
1942 1942
1943 1943 # Minimum time above which compilation time will be reported
1944 1944 tc_min = 0.1
1945 1945
1946 1946 try:
1947 1947 mode = 'eval'
1948 1948 t0 = clock()
1949 1949 code = compile(expr,'<timed eval>',mode)
1950 1950 tc = clock()-t0
1951 1951 except SyntaxError:
1952 1952 mode = 'exec'
1953 1953 t0 = clock()
1954 1954 code = compile(expr,'<timed exec>',mode)
1955 1955 tc = clock()-t0
1956 1956 # skew measurement as little as possible
1957 1957 glob = self.shell.user_ns
1958 1958 locs = self._magic_locals
1959 1959 clk = clock2
1960 1960 wtime = time.time
1961 1961 # time execution
1962 1962 wall_st = wtime()
1963 1963 if mode=='eval':
1964 1964 st = clk()
1965 1965 out = eval(code, glob, locs)
1966 1966 end = clk()
1967 1967 else:
1968 1968 st = clk()
1969 1969 exec code in glob, locs
1970 1970 end = clk()
1971 1971 out = None
1972 1972 wall_end = wtime()
1973 1973 # Compute actual times and report
1974 1974 wall_time = wall_end-wall_st
1975 1975 cpu_user = end[0]-st[0]
1976 1976 cpu_sys = end[1]-st[1]
1977 1977 cpu_tot = cpu_user+cpu_sys
1978 1978 print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \
1979 1979 (cpu_user,cpu_sys,cpu_tot)
1980 1980 print "Wall time: %.2f s" % wall_time
1981 1981 if tc > tc_min:
1982 1982 print "Compiler : %.2f s" % tc
1983 1983 return out
1984 1984
1985 1985 @skip_doctest
1986 1986 def magic_macro(self,parameter_s = ''):
1987 1987 """Define a macro for future re-execution. It accepts ranges of history,
1988 1988 filenames or string objects.
1989 1989
1990 1990 Usage:\\
1991 1991 %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
1992 1992
1993 1993 Options:
1994 1994
1995 1995 -r: use 'raw' input. By default, the 'processed' history is used,
1996 1996 so that magics are loaded in their transformed version to valid
1997 1997 Python. If this option is given, the raw input as typed as the
1998 1998 command line is used instead.
1999 1999
2000 2000 This will define a global variable called `name` which is a string
2001 2001 made of joining the slices and lines you specify (n1,n2,... numbers
2002 2002 above) from your input history into a single string. This variable
2003 2003 acts like an automatic function which re-executes those lines as if
2004 2004 you had typed them. You just type 'name' at the prompt and the code
2005 2005 executes.
2006 2006
2007 2007 The syntax for indicating input ranges is described in %history.
2008 2008
2009 2009 Note: as a 'hidden' feature, you can also use traditional python slice
2010 2010 notation, where N:M means numbers N through M-1.
2011 2011
2012 2012 For example, if your history contains (%hist prints it):
2013 2013
2014 2014 44: x=1
2015 2015 45: y=3
2016 2016 46: z=x+y
2017 2017 47: print x
2018 2018 48: a=5
2019 2019 49: print 'x',x,'y',y
2020 2020
2021 2021 you can create a macro with lines 44 through 47 (included) and line 49
2022 2022 called my_macro with:
2023 2023
2024 2024 In [55]: %macro my_macro 44-47 49
2025 2025
2026 2026 Now, typing `my_macro` (without quotes) will re-execute all this code
2027 2027 in one pass.
2028 2028
2029 2029 You don't need to give the line-numbers in order, and any given line
2030 2030 number can appear multiple times. You can assemble macros with any
2031 2031 lines from your input history in any order.
2032 2032
2033 2033 The macro is a simple object which holds its value in an attribute,
2034 2034 but IPython's display system checks for macros and executes them as
2035 2035 code instead of printing them when you type their name.
2036 2036
2037 2037 You can view a macro's contents by explicitly printing it with:
2038 2038
2039 2039 'print macro_name'.
2040 2040
2041 2041 """
2042 2042 opts,args = self.parse_options(parameter_s,'r',mode='list')
2043 2043 if not args: # List existing macros
2044 2044 return sorted(k for k,v in self.shell.user_ns.iteritems() if\
2045 2045 isinstance(v, Macro))
2046 2046 if len(args) == 1:
2047 2047 raise UsageError(
2048 2048 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
2049 2049 name, codefrom = args[0], " ".join(args[1:])
2050 2050
2051 2051 #print 'rng',ranges # dbg
2052 2052 try:
2053 2053 lines = self.shell.find_user_code(codefrom, 'r' in opts)
2054 2054 except (ValueError, TypeError) as e:
2055 2055 print e.args[0]
2056 2056 return
2057 2057 macro = Macro(lines)
2058 2058 self.shell.define_macro(name, macro)
2059 2059 print 'Macro `%s` created. To execute, type its name (without quotes).' % name
2060 2060 print '=== Macro contents: ==='
2061 2061 print macro,
2062 2062
2063 2063 def magic_save(self,parameter_s = ''):
2064 2064 """Save a set of lines or a macro to a given filename.
2065 2065
2066 2066 Usage:\\
2067 2067 %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
2068 2068
2069 2069 Options:
2070 2070
2071 2071 -r: use 'raw' input. By default, the 'processed' history is used,
2072 2072 so that magics are loaded in their transformed version to valid
2073 2073 Python. If this option is given, the raw input as typed as the
2074 2074 command line is used instead.
2075 2075
2076 2076 This function uses the same syntax as %history for input ranges,
2077 2077 then saves the lines to the filename you specify.
2078 2078
2079 2079 It adds a '.py' extension to the file if you don't do so yourself, and
2080 2080 it asks for confirmation before overwriting existing files."""
2081 2081
2082 2082 opts,args = self.parse_options(parameter_s,'r',mode='list')
2083 2083 fname, codefrom = args[0], " ".join(args[1:])
2084 2084 if not fname.endswith('.py'):
2085 2085 fname += '.py'
2086 2086 if os.path.isfile(fname):
2087 2087 ans = raw_input('File `%s` exists. Overwrite (y/[N])? ' % fname)
2088 2088 if ans.lower() not in ['y','yes']:
2089 2089 print 'Operation cancelled.'
2090 2090 return
2091 2091 try:
2092 2092 cmds = self.shell.find_user_code(codefrom, 'r' in opts)
2093 2093 except (TypeError, ValueError) as e:
2094 2094 print e.args[0]
2095 2095 return
2096 2096 if isinstance(cmds, unicode):
2097 2097 cmds = cmds.encode("utf-8")
2098 2098 with open(fname,'w') as f:
2099 2099 f.write("# coding: utf-8\n")
2100 2100 f.write(cmds)
2101 2101 print 'The following commands were written to file `%s`:' % fname
2102 2102 print cmds
2103 2103
2104 2104 def magic_pastebin(self, parameter_s = ''):
2105 2105 """Upload code to the 'Lodge it' paste bin, returning the URL."""
2106 2106 try:
2107 2107 code = self.shell.find_user_code(parameter_s)
2108 2108 except (ValueError, TypeError) as e:
2109 2109 print e.args[0]
2110 2110 return
2111 2111 pbserver = ServerProxy('http://paste.pocoo.org/xmlrpc/')
2112 2112 id = pbserver.pastes.newPaste("python", code)
2113 2113 return "http://paste.pocoo.org/show/" + id
2114 2114
2115 2115 def magic_loadpy(self, arg_s):
2116 2116 """Load a .py python script into the GUI console.
2117 2117
2118 2118 This magic command can either take a local filename or a url::
2119 2119
2120 2120 %loadpy myscript.py
2121 2121 %loadpy http://www.example.com/myscript.py
2122 2122 """
2123 2123 if not arg_s.endswith('.py'):
2124 2124 raise ValueError('%%load only works with .py files: %s' % arg_s)
2125 2125 if arg_s.startswith('http'):
2126 2126 import urllib2
2127 2127 response = urllib2.urlopen(arg_s)
2128 2128 content = response.read()
2129 2129 else:
2130 2130 content = open(arg_s).read()
2131 2131 self.set_next_input(content)
2132 2132
2133 2133 def _find_edit_target(self, args, opts, last_call):
2134 2134 """Utility method used by magic_edit to find what to edit."""
2135 2135
2136 2136 def make_filename(arg):
2137 2137 "Make a filename from the given args"
2138 2138 try:
2139 2139 filename = get_py_filename(arg)
2140 2140 except IOError:
2141 2141 # If it ends with .py but doesn't already exist, assume we want
2142 2142 # a new file.
2143 2143 if args.endswith('.py'):
2144 2144 filename = arg
2145 2145 else:
2146 2146 filename = None
2147 2147 return filename
2148 2148
2149 2149 # Set a few locals from the options for convenience:
2150 2150 opts_prev = 'p' in opts
2151 2151 opts_raw = 'r' in opts
2152 2152
2153 2153 # custom exceptions
2154 2154 class DataIsObject(Exception): pass
2155 2155
2156 2156 # Default line number value
2157 2157 lineno = opts.get('n',None)
2158 2158
2159 2159 if opts_prev:
2160 2160 args = '_%s' % last_call[0]
2161 2161 if not self.shell.user_ns.has_key(args):
2162 2162 args = last_call[1]
2163 2163
2164 2164 # use last_call to remember the state of the previous call, but don't
2165 2165 # let it be clobbered by successive '-p' calls.
2166 2166 try:
2167 2167 last_call[0] = self.shell.displayhook.prompt_count
2168 2168 if not opts_prev:
2169 2169 last_call[1] = parameter_s
2170 2170 except:
2171 2171 pass
2172 2172
2173 2173 # by default this is done with temp files, except when the given
2174 2174 # arg is a filename
2175 2175 use_temp = True
2176 2176
2177 2177 data = ''
2178 2178
2179 2179 # First, see if the arguments should be a filename.
2180 2180 filename = make_filename(args)
2181 2181 if filename:
2182 2182 use_temp = False
2183 2183 elif args:
2184 2184 # Mode where user specifies ranges of lines, like in %macro.
2185 2185 data = self.extract_input_lines(args, opts_raw)
2186 2186 if not data:
2187 2187 try:
2188 2188 # Load the parameter given as a variable. If not a string,
2189 2189 # process it as an object instead (below)
2190 2190
2191 2191 #print '*** args',args,'type',type(args) # dbg
2192 2192 data = eval(args, self.shell.user_ns)
2193 2193 if not isinstance(data, basestring):
2194 2194 raise DataIsObject
2195 2195
2196 2196 except (NameError,SyntaxError):
2197 2197 # given argument is not a variable, try as a filename
2198 2198 filename = make_filename(args)
2199 2199 if filename is None:
2200 2200 warn("Argument given (%s) can't be found as a variable "
2201 2201 "or as a filename." % args)
2202 2202 return
2203 2203 use_temp = False
2204 2204
2205 2205 except DataIsObject:
2206 2206 # macros have a special edit function
2207 2207 if isinstance(data, Macro):
2208 2208 raise MacroToEdit(data)
2209 2209
2210 2210 # For objects, try to edit the file where they are defined
2211 2211 try:
2212 2212 filename = inspect.getabsfile(data)
2213 2213 if 'fakemodule' in filename.lower() and inspect.isclass(data):
2214 2214 # class created by %edit? Try to find source
2215 2215 # by looking for method definitions instead, the
2216 2216 # __module__ in those classes is FakeModule.
2217 2217 attrs = [getattr(data, aname) for aname in dir(data)]
2218 2218 for attr in attrs:
2219 2219 if not inspect.ismethod(attr):
2220 2220 continue
2221 2221 filename = inspect.getabsfile(attr)
2222 2222 if filename and 'fakemodule' not in filename.lower():
2223 2223 # change the attribute to be the edit target instead
2224 2224 data = attr
2225 2225 break
2226 2226
2227 2227 datafile = 1
2228 2228 except TypeError:
2229 2229 filename = make_filename(args)
2230 2230 datafile = 1
2231 2231 warn('Could not find file where `%s` is defined.\n'
2232 2232 'Opening a file named `%s`' % (args,filename))
2233 2233 # Now, make sure we can actually read the source (if it was in
2234 2234 # a temp file it's gone by now).
2235 2235 if datafile:
2236 2236 try:
2237 2237 if lineno is None:
2238 2238 lineno = inspect.getsourcelines(data)[1]
2239 2239 except IOError:
2240 2240 filename = make_filename(args)
2241 2241 if filename is None:
2242 2242 warn('The file `%s` where `%s` was defined cannot '
2243 2243 'be read.' % (filename,data))
2244 2244 return
2245 2245 use_temp = False
2246 2246
2247 2247 if use_temp:
2248 2248 filename = self.shell.mktempfile(data)
2249 2249 print 'IPython will make a temporary file named:',filename
2250 2250
2251 2251 return filename, lineno, use_temp
2252 2252
2253 2253 def _edit_macro(self,mname,macro):
2254 2254 """open an editor with the macro data in a file"""
2255 2255 filename = self.shell.mktempfile(macro.value)
2256 2256 self.shell.hooks.editor(filename)
2257 2257
2258 2258 # and make a new macro object, to replace the old one
2259 2259 mfile = open(filename)
2260 2260 mvalue = mfile.read()
2261 2261 mfile.close()
2262 2262 self.shell.user_ns[mname] = Macro(mvalue)
2263 2263
2264 2264 def magic_ed(self,parameter_s=''):
2265 2265 """Alias to %edit."""
2266 2266 return self.magic_edit(parameter_s)
2267 2267
2268 2268 @skip_doctest
2269 2269 def magic_edit(self,parameter_s='',last_call=['','']):
2270 2270 """Bring up an editor and execute the resulting code.
2271 2271
2272 2272 Usage:
2273 2273 %edit [options] [args]
2274 2274
2275 2275 %edit runs IPython's editor hook. The default version of this hook is
2276 2276 set to call the __IPYTHON__.rc.editor command. This is read from your
2277 2277 environment variable $EDITOR. If this isn't found, it will default to
2278 2278 vi under Linux/Unix and to notepad under Windows. See the end of this
2279 2279 docstring for how to change the editor hook.
2280 2280
2281 2281 You can also set the value of this editor via the command line option
2282 2282 '-editor' or in your ipythonrc file. This is useful if you wish to use
2283 2283 specifically for IPython an editor different from your typical default
2284 2284 (and for Windows users who typically don't set environment variables).
2285 2285
2286 2286 This command allows you to conveniently edit multi-line code right in
2287 2287 your IPython session.
2288 2288
2289 2289 If called without arguments, %edit opens up an empty editor with a
2290 2290 temporary file and will execute the contents of this file when you
2291 2291 close it (don't forget to save it!).
2292 2292
2293 2293
2294 2294 Options:
2295 2295
2296 2296 -n <number>: open the editor at a specified line number. By default,
2297 2297 the IPython editor hook uses the unix syntax 'editor +N filename', but
2298 2298 you can configure this by providing your own modified hook if your
2299 2299 favorite editor supports line-number specifications with a different
2300 2300 syntax.
2301 2301
2302 2302 -p: this will call the editor with the same data as the previous time
2303 2303 it was used, regardless of how long ago (in your current session) it
2304 2304 was.
2305 2305
2306 2306 -r: use 'raw' input. This option only applies to input taken from the
2307 2307 user's history. By default, the 'processed' history is used, so that
2308 2308 magics are loaded in their transformed version to valid Python. If
2309 2309 this option is given, the raw input as typed as the command line is
2310 2310 used instead. When you exit the editor, it will be executed by
2311 2311 IPython's own processor.
2312 2312
2313 2313 -x: do not execute the edited code immediately upon exit. This is
2314 2314 mainly useful if you are editing programs which need to be called with
2315 2315 command line arguments, which you can then do using %run.
2316 2316
2317 2317
2318 2318 Arguments:
2319 2319
2320 2320 If arguments are given, the following possibilites exist:
2321 2321
2322 2322 - If the argument is a filename, IPython will load that into the
2323 2323 editor. It will execute its contents with execfile() when you exit,
2324 2324 loading any code in the file into your interactive namespace.
2325 2325
2326 2326 - The arguments are ranges of input history, e.g. "7 ~1/4-6".
2327 2327 The syntax is the same as in the %history magic.
2328 2328
2329 2329 - If the argument is a string variable, its contents are loaded
2330 2330 into the editor. You can thus edit any string which contains
2331 2331 python code (including the result of previous edits).
2332 2332
2333 2333 - If the argument is the name of an object (other than a string),
2334 2334 IPython will try to locate the file where it was defined and open the
2335 2335 editor at the point where it is defined. You can use `%edit function`
2336 2336 to load an editor exactly at the point where 'function' is defined,
2337 2337 edit it and have the file be executed automatically.
2338 2338
2339 2339 If the object is a macro (see %macro for details), this opens up your
2340 2340 specified editor with a temporary file containing the macro's data.
2341 2341 Upon exit, the macro is reloaded with the contents of the file.
2342 2342
2343 2343 Note: opening at an exact line is only supported under Unix, and some
2344 2344 editors (like kedit and gedit up to Gnome 2.8) do not understand the
2345 2345 '+NUMBER' parameter necessary for this feature. Good editors like
2346 2346 (X)Emacs, vi, jed, pico and joe all do.
2347 2347
2348 2348 After executing your code, %edit will return as output the code you
2349 2349 typed in the editor (except when it was an existing file). This way
2350 2350 you can reload the code in further invocations of %edit as a variable,
2351 2351 via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
2352 2352 the output.
2353 2353
2354 2354 Note that %edit is also available through the alias %ed.
2355 2355
2356 2356 This is an example of creating a simple function inside the editor and
2357 2357 then modifying it. First, start up the editor:
2358 2358
2359 2359 In [1]: ed
2360 2360 Editing... done. Executing edited code...
2361 2361 Out[1]: 'def foo():n print "foo() was defined in an editing session"n'
2362 2362
2363 2363 We can then call the function foo():
2364 2364
2365 2365 In [2]: foo()
2366 2366 foo() was defined in an editing session
2367 2367
2368 2368 Now we edit foo. IPython automatically loads the editor with the
2369 2369 (temporary) file where foo() was previously defined:
2370 2370
2371 2371 In [3]: ed foo
2372 2372 Editing... done. Executing edited code...
2373 2373
2374 2374 And if we call foo() again we get the modified version:
2375 2375
2376 2376 In [4]: foo()
2377 2377 foo() has now been changed!
2378 2378
2379 2379 Here is an example of how to edit a code snippet successive
2380 2380 times. First we call the editor:
2381 2381
2382 2382 In [5]: ed
2383 2383 Editing... done. Executing edited code...
2384 2384 hello
2385 2385 Out[5]: "print 'hello'n"
2386 2386
2387 2387 Now we call it again with the previous output (stored in _):
2388 2388
2389 2389 In [6]: ed _
2390 2390 Editing... done. Executing edited code...
2391 2391 hello world
2392 2392 Out[6]: "print 'hello world'n"
2393 2393
2394 2394 Now we call it with the output #8 (stored in _8, also as Out[8]):
2395 2395
2396 2396 In [7]: ed _8
2397 2397 Editing... done. Executing edited code...
2398 2398 hello again
2399 2399 Out[7]: "print 'hello again'n"
2400 2400
2401 2401
2402 2402 Changing the default editor hook:
2403 2403
2404 2404 If you wish to write your own editor hook, you can put it in a
2405 2405 configuration file which you load at startup time. The default hook
2406 2406 is defined in the IPython.core.hooks module, and you can use that as a
2407 2407 starting example for further modifications. That file also has
2408 2408 general instructions on how to set a new hook for use once you've
2409 2409 defined it."""
2410 2410 opts,args = self.parse_options(parameter_s,'prxn:')
2411 2411
2412 2412 try:
2413 2413 filename, lineno, is_temp = self._find_edit_target(args, opts, last_call)
2414 2414 except MacroToEdit as e:
2415 2415 self._edit_macro(args, e.args[0])
2416 2416 return
2417 2417
2418 2418 # do actual editing here
2419 2419 print 'Editing...',
2420 2420 sys.stdout.flush()
2421 2421 try:
2422 2422 # Quote filenames that may have spaces in them
2423 2423 if ' ' in filename:
2424 2424 filename = "'%s'" % filename
2425 2425 self.shell.hooks.editor(filename,lineno)
2426 2426 except TryNext:
2427 2427 warn('Could not open editor')
2428 2428 return
2429 2429
2430 2430 # XXX TODO: should this be generalized for all string vars?
2431 2431 # For now, this is special-cased to blocks created by cpaste
2432 2432 if args.strip() == 'pasted_block':
2433 2433 self.shell.user_ns['pasted_block'] = file_read(filename)
2434 2434
2435 2435 if 'x' in opts: # -x prevents actual execution
2436 2436 print
2437 2437 else:
2438 2438 print 'done. Executing edited code...'
2439 2439 if 'r' in opts: # Untranslated IPython code
2440 2440 self.shell.run_cell(file_read(filename),
2441 2441 store_history=False)
2442 2442 else:
2443 2443 self.shell.safe_execfile(filename,self.shell.user_ns,
2444 2444 self.shell.user_ns)
2445 2445
2446 2446 if is_temp:
2447 2447 try:
2448 2448 return open(filename).read()
2449 2449 except IOError,msg:
2450 2450 if msg.filename == filename:
2451 2451 warn('File not found. Did you forget to save?')
2452 2452 return
2453 2453 else:
2454 2454 self.shell.showtraceback()
2455 2455
2456 2456 def magic_xmode(self,parameter_s = ''):
2457 2457 """Switch modes for the exception handlers.
2458 2458
2459 2459 Valid modes: Plain, Context and Verbose.
2460 2460
2461 2461 If called without arguments, acts as a toggle."""
2462 2462
2463 2463 def xmode_switch_err(name):
2464 2464 warn('Error changing %s exception modes.\n%s' %
2465 2465 (name,sys.exc_info()[1]))
2466 2466
2467 2467 shell = self.shell
2468 2468 new_mode = parameter_s.strip().capitalize()
2469 2469 try:
2470 2470 shell.InteractiveTB.set_mode(mode=new_mode)
2471 2471 print 'Exception reporting mode:',shell.InteractiveTB.mode
2472 2472 except:
2473 2473 xmode_switch_err('user')
2474 2474
2475 2475 def magic_colors(self,parameter_s = ''):
2476 2476 """Switch color scheme for prompts, info system and exception handlers.
2477 2477
2478 2478 Currently implemented schemes: NoColor, Linux, LightBG.
2479 2479
2480 2480 Color scheme names are not case-sensitive.
2481 2481
2482 2482 Examples
2483 2483 --------
2484 2484 To get a plain black and white terminal::
2485 2485
2486 2486 %colors nocolor
2487 2487 """
2488 2488
2489 2489 def color_switch_err(name):
2490 2490 warn('Error changing %s color schemes.\n%s' %
2491 2491 (name,sys.exc_info()[1]))
2492 2492
2493 2493
2494 2494 new_scheme = parameter_s.strip()
2495 2495 if not new_scheme:
2496 2496 raise UsageError(
2497 2497 "%colors: you must specify a color scheme. See '%colors?'")
2498 2498 return
2499 2499 # local shortcut
2500 2500 shell = self.shell
2501 2501
2502 2502 import IPython.utils.rlineimpl as readline
2503 2503
2504 2504 if not readline.have_readline and sys.platform == "win32":
2505 2505 msg = """\
2506 2506 Proper color support under MS Windows requires the pyreadline library.
2507 2507 You can find it at:
2508 2508 http://ipython.scipy.org/moin/PyReadline/Intro
2509 2509 Gary's readline needs the ctypes module, from:
2510 2510 http://starship.python.net/crew/theller/ctypes
2511 2511 (Note that ctypes is already part of Python versions 2.5 and newer).
2512 2512
2513 2513 Defaulting color scheme to 'NoColor'"""
2514 2514 new_scheme = 'NoColor'
2515 2515 warn(msg)
2516 2516
2517 2517 # readline option is 0
2518 2518 if not shell.has_readline:
2519 2519 new_scheme = 'NoColor'
2520 2520
2521 2521 # Set prompt colors
2522 2522 try:
2523 2523 shell.displayhook.set_colors(new_scheme)
2524 2524 except:
2525 2525 color_switch_err('prompt')
2526 2526 else:
2527 2527 shell.colors = \
2528 2528 shell.displayhook.color_table.active_scheme_name
2529 2529 # Set exception colors
2530 2530 try:
2531 2531 shell.InteractiveTB.set_colors(scheme = new_scheme)
2532 2532 shell.SyntaxTB.set_colors(scheme = new_scheme)
2533 2533 except:
2534 2534 color_switch_err('exception')
2535 2535
2536 2536 # Set info (for 'object?') colors
2537 2537 if shell.color_info:
2538 2538 try:
2539 2539 shell.inspector.set_active_scheme(new_scheme)
2540 2540 except:
2541 2541 color_switch_err('object inspector')
2542 2542 else:
2543 2543 shell.inspector.set_active_scheme('NoColor')
2544 2544
2545 2545 def magic_pprint(self, parameter_s=''):
2546 2546 """Toggle pretty printing on/off."""
2547 2547 ptformatter = self.shell.display_formatter.formatters['text/plain']
2548 2548 ptformatter.pprint = bool(1 - ptformatter.pprint)
2549 2549 print 'Pretty printing has been turned', \
2550 2550 ['OFF','ON'][ptformatter.pprint]
2551 2551
2552 2552 #......................................................................
2553 2553 # Functions to implement unix shell-type things
2554 2554
2555 2555 @skip_doctest
2556 2556 def magic_alias(self, parameter_s = ''):
2557 2557 """Define an alias for a system command.
2558 2558
2559 2559 '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
2560 2560
2561 2561 Then, typing 'alias_name params' will execute the system command 'cmd
2562 2562 params' (from your underlying operating system).
2563 2563
2564 2564 Aliases have lower precedence than magic functions and Python normal
2565 2565 variables, so if 'foo' is both a Python variable and an alias, the
2566 2566 alias can not be executed until 'del foo' removes the Python variable.
2567 2567
2568 2568 You can use the %l specifier in an alias definition to represent the
2569 2569 whole line when the alias is called. For example:
2570 2570
2571 2571 In [2]: alias bracket echo "Input in brackets: <%l>"
2572 2572 In [3]: bracket hello world
2573 2573 Input in brackets: <hello world>
2574 2574
2575 2575 You can also define aliases with parameters using %s specifiers (one
2576 2576 per parameter):
2577 2577
2578 2578 In [1]: alias parts echo first %s second %s
2579 2579 In [2]: %parts A B
2580 2580 first A second B
2581 2581 In [3]: %parts A
2582 2582 Incorrect number of arguments: 2 expected.
2583 2583 parts is an alias to: 'echo first %s second %s'
2584 2584
2585 2585 Note that %l and %s are mutually exclusive. You can only use one or
2586 2586 the other in your aliases.
2587 2587
2588 2588 Aliases expand Python variables just like system calls using ! or !!
2589 2589 do: all expressions prefixed with '$' get expanded. For details of
2590 2590 the semantic rules, see PEP-215:
2591 2591 http://www.python.org/peps/pep-0215.html. This is the library used by
2592 2592 IPython for variable expansion. If you want to access a true shell
2593 2593 variable, an extra $ is necessary to prevent its expansion by IPython:
2594 2594
2595 2595 In [6]: alias show echo
2596 2596 In [7]: PATH='A Python string'
2597 2597 In [8]: show $PATH
2598 2598 A Python string
2599 2599 In [9]: show $$PATH
2600 2600 /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
2601 2601
2602 2602 You can use the alias facility to acess all of $PATH. See the %rehash
2603 2603 and %rehashx functions, which automatically create aliases for the
2604 2604 contents of your $PATH.
2605 2605
2606 2606 If called with no parameters, %alias prints the current alias table."""
2607 2607
2608 2608 par = parameter_s.strip()
2609 2609 if not par:
2610 2610 stored = self.db.get('stored_aliases', {} )
2611 2611 aliases = sorted(self.shell.alias_manager.aliases)
2612 2612 # for k, v in stored:
2613 2613 # atab.append(k, v[0])
2614 2614
2615 2615 print "Total number of aliases:", len(aliases)
2616 2616 sys.stdout.flush()
2617 2617 return aliases
2618 2618
2619 2619 # Now try to define a new one
2620 2620 try:
2621 2621 alias,cmd = par.split(None, 1)
2622 2622 except:
2623 2623 print oinspect.getdoc(self.magic_alias)
2624 2624 else:
2625 2625 self.shell.alias_manager.soft_define_alias(alias, cmd)
2626 2626 # end magic_alias
2627 2627
2628 2628 def magic_unalias(self, parameter_s = ''):
2629 2629 """Remove an alias"""
2630 2630
2631 2631 aname = parameter_s.strip()
2632 2632 self.shell.alias_manager.undefine_alias(aname)
2633 2633 stored = self.db.get('stored_aliases', {} )
2634 2634 if aname in stored:
2635 2635 print "Removing %stored alias",aname
2636 2636 del stored[aname]
2637 2637 self.db['stored_aliases'] = stored
2638 2638
2639 2639 def magic_rehashx(self, parameter_s = ''):
2640 2640 """Update the alias table with all executable files in $PATH.
2641 2641
2642 2642 This version explicitly checks that every entry in $PATH is a file
2643 2643 with execute access (os.X_OK), so it is much slower than %rehash.
2644 2644
2645 2645 Under Windows, it checks executability as a match agains a
2646 2646 '|'-separated string of extensions, stored in the IPython config
2647 2647 variable win_exec_ext. This defaults to 'exe|com|bat'.
2648 2648
2649 2649 This function also resets the root module cache of module completer,
2650 2650 used on slow filesystems.
2651 2651 """
2652 2652 from IPython.core.alias import InvalidAliasError
2653 2653
2654 2654 # for the benefit of module completer in ipy_completers.py
2655 2655 del self.db['rootmodules']
2656 2656
2657 2657 path = [os.path.abspath(os.path.expanduser(p)) for p in
2658 2658 os.environ.get('PATH','').split(os.pathsep)]
2659 2659 path = filter(os.path.isdir,path)
2660 2660
2661 2661 syscmdlist = []
2662 2662 # Now define isexec in a cross platform manner.
2663 2663 if os.name == 'posix':
2664 2664 isexec = lambda fname:os.path.isfile(fname) and \
2665 2665 os.access(fname,os.X_OK)
2666 2666 else:
2667 2667 try:
2668 2668 winext = os.environ['pathext'].replace(';','|').replace('.','')
2669 2669 except KeyError:
2670 2670 winext = 'exe|com|bat|py'
2671 2671 if 'py' not in winext:
2672 2672 winext += '|py'
2673 2673 execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
2674 2674 isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
2675 2675 savedir = os.getcwd()
2676 2676
2677 2677 # Now walk the paths looking for executables to alias.
2678 2678 try:
2679 2679 # write the whole loop for posix/Windows so we don't have an if in
2680 2680 # the innermost part
2681 2681 if os.name == 'posix':
2682 2682 for pdir in path:
2683 2683 os.chdir(pdir)
2684 2684 for ff in os.listdir(pdir):
2685 2685 if isexec(ff):
2686 2686 try:
2687 2687 # Removes dots from the name since ipython
2688 2688 # will assume names with dots to be python.
2689 2689 self.shell.alias_manager.define_alias(
2690 2690 ff.replace('.',''), ff)
2691 2691 except InvalidAliasError:
2692 2692 pass
2693 2693 else:
2694 2694 syscmdlist.append(ff)
2695 2695 else:
2696 2696 no_alias = self.shell.alias_manager.no_alias
2697 2697 for pdir in path:
2698 2698 os.chdir(pdir)
2699 2699 for ff in os.listdir(pdir):
2700 2700 base, ext = os.path.splitext(ff)
2701 2701 if isexec(ff) and base.lower() not in no_alias:
2702 2702 if ext.lower() == '.exe':
2703 2703 ff = base
2704 2704 try:
2705 2705 # Removes dots from the name since ipython
2706 2706 # will assume names with dots to be python.
2707 2707 self.shell.alias_manager.define_alias(
2708 2708 base.lower().replace('.',''), ff)
2709 2709 except InvalidAliasError:
2710 2710 pass
2711 2711 syscmdlist.append(ff)
2712 2712 db = self.db
2713 2713 db['syscmdlist'] = syscmdlist
2714 2714 finally:
2715 2715 os.chdir(savedir)
2716 2716
2717 2717 @skip_doctest
2718 2718 def magic_pwd(self, parameter_s = ''):
2719 2719 """Return the current working directory path.
2720 2720
2721 2721 Examples
2722 2722 --------
2723 2723 ::
2724 2724
2725 2725 In [9]: pwd
2726 2726 Out[9]: '/home/tsuser/sprint/ipython'
2727 2727 """
2728 2728 return os.getcwd()
2729 2729
2730 2730 @skip_doctest
2731 2731 def magic_cd(self, parameter_s=''):
2732 2732 """Change the current working directory.
2733 2733
2734 2734 This command automatically maintains an internal list of directories
2735 2735 you visit during your IPython session, in the variable _dh. The
2736 2736 command %dhist shows this history nicely formatted. You can also
2737 2737 do 'cd -<tab>' to see directory history conveniently.
2738 2738
2739 2739 Usage:
2740 2740
2741 2741 cd 'dir': changes to directory 'dir'.
2742 2742
2743 2743 cd -: changes to the last visited directory.
2744 2744
2745 2745 cd -<n>: changes to the n-th directory in the directory history.
2746 2746
2747 2747 cd --foo: change to directory that matches 'foo' in history
2748 2748
2749 2749 cd -b <bookmark_name>: jump to a bookmark set by %bookmark
2750 2750 (note: cd <bookmark_name> is enough if there is no
2751 2751 directory <bookmark_name>, but a bookmark with the name exists.)
2752 2752 'cd -b <tab>' allows you to tab-complete bookmark names.
2753 2753
2754 2754 Options:
2755 2755
2756 2756 -q: quiet. Do not print the working directory after the cd command is
2757 2757 executed. By default IPython's cd command does print this directory,
2758 2758 since the default prompts do not display path information.
2759 2759
2760 2760 Note that !cd doesn't work for this purpose because the shell where
2761 2761 !command runs is immediately discarded after executing 'command'.
2762 2762
2763 2763 Examples
2764 2764 --------
2765 2765 ::
2766 2766
2767 2767 In [10]: cd parent/child
2768 2768 /home/tsuser/parent/child
2769 2769 """
2770 2770
2771 2771 parameter_s = parameter_s.strip()
2772 2772 #bkms = self.shell.persist.get("bookmarks",{})
2773 2773
2774 2774 oldcwd = os.getcwd()
2775 2775 numcd = re.match(r'(-)(\d+)$',parameter_s)
2776 2776 # jump in directory history by number
2777 2777 if numcd:
2778 2778 nn = int(numcd.group(2))
2779 2779 try:
2780 2780 ps = self.shell.user_ns['_dh'][nn]
2781 2781 except IndexError:
2782 2782 print 'The requested directory does not exist in history.'
2783 2783 return
2784 2784 else:
2785 2785 opts = {}
2786 2786 elif parameter_s.startswith('--'):
2787 2787 ps = None
2788 2788 fallback = None
2789 2789 pat = parameter_s[2:]
2790 2790 dh = self.shell.user_ns['_dh']
2791 2791 # first search only by basename (last component)
2792 2792 for ent in reversed(dh):
2793 2793 if pat in os.path.basename(ent) and os.path.isdir(ent):
2794 2794 ps = ent
2795 2795 break
2796 2796
2797 2797 if fallback is None and pat in ent and os.path.isdir(ent):
2798 2798 fallback = ent
2799 2799
2800 2800 # if we have no last part match, pick the first full path match
2801 2801 if ps is None:
2802 2802 ps = fallback
2803 2803
2804 2804 if ps is None:
2805 2805 print "No matching entry in directory history"
2806 2806 return
2807 2807 else:
2808 2808 opts = {}
2809 2809
2810 2810
2811 2811 else:
2812 2812 #turn all non-space-escaping backslashes to slashes,
2813 2813 # for c:\windows\directory\names\
2814 2814 parameter_s = re.sub(r'\\(?! )','/', parameter_s)
2815 2815 opts,ps = self.parse_options(parameter_s,'qb',mode='string')
2816 2816 # jump to previous
2817 2817 if ps == '-':
2818 2818 try:
2819 2819 ps = self.shell.user_ns['_dh'][-2]
2820 2820 except IndexError:
2821 2821 raise UsageError('%cd -: No previous directory to change to.')
2822 2822 # jump to bookmark if needed
2823 2823 else:
2824 2824 if not os.path.isdir(ps) or opts.has_key('b'):
2825 2825 bkms = self.db.get('bookmarks', {})
2826 2826
2827 2827 if bkms.has_key(ps):
2828 2828 target = bkms[ps]
2829 2829 print '(bookmark:%s) -> %s' % (ps,target)
2830 2830 ps = target
2831 2831 else:
2832 2832 if opts.has_key('b'):
2833 2833 raise UsageError("Bookmark '%s' not found. "
2834 2834 "Use '%%bookmark -l' to see your bookmarks." % ps)
2835 2835
2836 2836 # at this point ps should point to the target dir
2837 2837 if ps:
2838 2838 try:
2839 2839 os.chdir(os.path.expanduser(ps))
2840 2840 if hasattr(self.shell, 'term_title') and self.shell.term_title:
2841 2841 set_term_title('IPython: ' + abbrev_cwd())
2842 2842 except OSError:
2843 2843 print sys.exc_info()[1]
2844 2844 else:
2845 2845 cwd = os.getcwd()
2846 2846 dhist = self.shell.user_ns['_dh']
2847 2847 if oldcwd != cwd:
2848 2848 dhist.append(cwd)
2849 2849 self.db['dhist'] = compress_dhist(dhist)[-100:]
2850 2850
2851 2851 else:
2852 2852 os.chdir(self.shell.home_dir)
2853 2853 if hasattr(self.shell, 'term_title') and self.shell.term_title:
2854 2854 set_term_title('IPython: ' + '~')
2855 2855 cwd = os.getcwd()
2856 2856 dhist = self.shell.user_ns['_dh']
2857 2857
2858 2858 if oldcwd != cwd:
2859 2859 dhist.append(cwd)
2860 2860 self.db['dhist'] = compress_dhist(dhist)[-100:]
2861 2861 if not 'q' in opts and self.shell.user_ns['_dh']:
2862 2862 print self.shell.user_ns['_dh'][-1]
2863 2863
2864 2864
2865 2865 def magic_env(self, parameter_s=''):
2866 2866 """List environment variables."""
2867 2867
2868 2868 return os.environ.data
2869 2869
2870 2870 def magic_pushd(self, parameter_s=''):
2871 2871 """Place the current dir on stack and change directory.
2872 2872
2873 2873 Usage:\\
2874 2874 %pushd ['dirname']
2875 2875 """
2876 2876
2877 2877 dir_s = self.shell.dir_stack
2878 2878 tgt = os.path.expanduser(parameter_s)
2879 2879 cwd = os.getcwd().replace(self.home_dir,'~')
2880 2880 if tgt:
2881 2881 self.magic_cd(parameter_s)
2882 2882 dir_s.insert(0,cwd)
2883 2883 return self.magic_dirs()
2884 2884
2885 2885 def magic_popd(self, parameter_s=''):
2886 2886 """Change to directory popped off the top of the stack.
2887 2887 """
2888 2888 if not self.shell.dir_stack:
2889 2889 raise UsageError("%popd on empty stack")
2890 2890 top = self.shell.dir_stack.pop(0)
2891 2891 self.magic_cd(top)
2892 2892 print "popd ->",top
2893 2893
2894 2894 def magic_dirs(self, parameter_s=''):
2895 2895 """Return the current directory stack."""
2896 2896
2897 2897 return self.shell.dir_stack
2898 2898
2899 2899 def magic_dhist(self, parameter_s=''):
2900 2900 """Print your history of visited directories.
2901 2901
2902 2902 %dhist -> print full history\\
2903 2903 %dhist n -> print last n entries only\\
2904 2904 %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\
2905 2905
2906 2906 This history is automatically maintained by the %cd command, and
2907 2907 always available as the global list variable _dh. You can use %cd -<n>
2908 2908 to go to directory number <n>.
2909 2909
2910 2910 Note that most of time, you should view directory history by entering
2911 2911 cd -<TAB>.
2912 2912
2913 2913 """
2914 2914
2915 2915 dh = self.shell.user_ns['_dh']
2916 2916 if parameter_s:
2917 2917 try:
2918 2918 args = map(int,parameter_s.split())
2919 2919 except:
2920 2920 self.arg_err(Magic.magic_dhist)
2921 2921 return
2922 2922 if len(args) == 1:
2923 2923 ini,fin = max(len(dh)-(args[0]),0),len(dh)
2924 2924 elif len(args) == 2:
2925 2925 ini,fin = args
2926 2926 else:
2927 2927 self.arg_err(Magic.magic_dhist)
2928 2928 return
2929 2929 else:
2930 2930 ini,fin = 0,len(dh)
2931 2931 nlprint(dh,
2932 2932 header = 'Directory history (kept in _dh)',
2933 2933 start=ini,stop=fin)
2934 2934
2935 2935 @skip_doctest
2936 2936 def magic_sc(self, parameter_s=''):
2937 2937 """Shell capture - execute a shell command and capture its output.
2938 2938
2939 2939 DEPRECATED. Suboptimal, retained for backwards compatibility.
2940 2940
2941 2941 You should use the form 'var = !command' instead. Example:
2942 2942
2943 2943 "%sc -l myfiles = ls ~" should now be written as
2944 2944
2945 2945 "myfiles = !ls ~"
2946 2946
2947 2947 myfiles.s, myfiles.l and myfiles.n still apply as documented
2948 2948 below.
2949 2949
2950 2950 --
2951 2951 %sc [options] varname=command
2952 2952
2953 2953 IPython will run the given command using commands.getoutput(), and
2954 2954 will then update the user's interactive namespace with a variable
2955 2955 called varname, containing the value of the call. Your command can
2956 2956 contain shell wildcards, pipes, etc.
2957 2957
2958 2958 The '=' sign in the syntax is mandatory, and the variable name you
2959 2959 supply must follow Python's standard conventions for valid names.
2960 2960
2961 2961 (A special format without variable name exists for internal use)
2962 2962
2963 2963 Options:
2964 2964
2965 2965 -l: list output. Split the output on newlines into a list before
2966 2966 assigning it to the given variable. By default the output is stored
2967 2967 as a single string.
2968 2968
2969 2969 -v: verbose. Print the contents of the variable.
2970 2970
2971 2971 In most cases you should not need to split as a list, because the
2972 2972 returned value is a special type of string which can automatically
2973 2973 provide its contents either as a list (split on newlines) or as a
2974 2974 space-separated string. These are convenient, respectively, either
2975 2975 for sequential processing or to be passed to a shell command.
2976 2976
2977 2977 For example:
2978 2978
2979 2979 # all-random
2980 2980
2981 2981 # Capture into variable a
2982 2982 In [1]: sc a=ls *py
2983 2983
2984 2984 # a is a string with embedded newlines
2985 2985 In [2]: a
2986 2986 Out[2]: 'setup.py\\nwin32_manual_post_install.py'
2987 2987
2988 2988 # which can be seen as a list:
2989 2989 In [3]: a.l
2990 2990 Out[3]: ['setup.py', 'win32_manual_post_install.py']
2991 2991
2992 2992 # or as a whitespace-separated string:
2993 2993 In [4]: a.s
2994 2994 Out[4]: 'setup.py win32_manual_post_install.py'
2995 2995
2996 2996 # a.s is useful to pass as a single command line:
2997 2997 In [5]: !wc -l $a.s
2998 2998 146 setup.py
2999 2999 130 win32_manual_post_install.py
3000 3000 276 total
3001 3001
3002 3002 # while the list form is useful to loop over:
3003 3003 In [6]: for f in a.l:
3004 3004 ...: !wc -l $f
3005 3005 ...:
3006 3006 146 setup.py
3007 3007 130 win32_manual_post_install.py
3008 3008
3009 3009 Similiarly, the lists returned by the -l option are also special, in
3010 3010 the sense that you can equally invoke the .s attribute on them to
3011 3011 automatically get a whitespace-separated string from their contents:
3012 3012
3013 3013 In [7]: sc -l b=ls *py
3014 3014
3015 3015 In [8]: b
3016 3016 Out[8]: ['setup.py', 'win32_manual_post_install.py']
3017 3017
3018 3018 In [9]: b.s
3019 3019 Out[9]: 'setup.py win32_manual_post_install.py'
3020 3020
3021 3021 In summary, both the lists and strings used for ouptut capture have
3022 3022 the following special attributes:
3023 3023
3024 3024 .l (or .list) : value as list.
3025 3025 .n (or .nlstr): value as newline-separated string.
3026 3026 .s (or .spstr): value as space-separated string.
3027 3027 """
3028 3028
3029 3029 opts,args = self.parse_options(parameter_s,'lv')
3030 3030 # Try to get a variable name and command to run
3031 3031 try:
3032 3032 # the variable name must be obtained from the parse_options
3033 3033 # output, which uses shlex.split to strip options out.
3034 3034 var,_ = args.split('=',1)
3035 3035 var = var.strip()
3036 3036 # But the the command has to be extracted from the original input
3037 3037 # parameter_s, not on what parse_options returns, to avoid the
3038 3038 # quote stripping which shlex.split performs on it.
3039 3039 _,cmd = parameter_s.split('=',1)
3040 3040 except ValueError:
3041 3041 var,cmd = '',''
3042 3042 # If all looks ok, proceed
3043 3043 split = 'l' in opts
3044 3044 out = self.shell.getoutput(cmd, split=split)
3045 3045 if opts.has_key('v'):
3046 3046 print '%s ==\n%s' % (var,pformat(out))
3047 3047 if var:
3048 3048 self.shell.user_ns.update({var:out})
3049 3049 else:
3050 3050 return out
3051 3051
3052 3052 def magic_sx(self, parameter_s=''):
3053 3053 """Shell execute - run a shell command and capture its output.
3054 3054
3055 3055 %sx command
3056 3056
3057 3057 IPython will run the given command using commands.getoutput(), and
3058 3058 return the result formatted as a list (split on '\\n'). Since the
3059 3059 output is _returned_, it will be stored in ipython's regular output
3060 3060 cache Out[N] and in the '_N' automatic variables.
3061 3061
3062 3062 Notes:
3063 3063
3064 3064 1) If an input line begins with '!!', then %sx is automatically
3065 3065 invoked. That is, while:
3066 3066 !ls
3067 3067 causes ipython to simply issue system('ls'), typing
3068 3068 !!ls
3069 3069 is a shorthand equivalent to:
3070 3070 %sx ls
3071 3071
3072 3072 2) %sx differs from %sc in that %sx automatically splits into a list,
3073 3073 like '%sc -l'. The reason for this is to make it as easy as possible
3074 3074 to process line-oriented shell output via further python commands.
3075 3075 %sc is meant to provide much finer control, but requires more
3076 3076 typing.
3077 3077
3078 3078 3) Just like %sc -l, this is a list with special attributes:
3079 3079
3080 3080 .l (or .list) : value as list.
3081 3081 .n (or .nlstr): value as newline-separated string.
3082 3082 .s (or .spstr): value as whitespace-separated string.
3083 3083
3084 3084 This is very useful when trying to use such lists as arguments to
3085 3085 system commands."""
3086 3086
3087 3087 if parameter_s:
3088 3088 return self.shell.getoutput(parameter_s)
3089 3089
3090 3090
3091 3091 def magic_bookmark(self, parameter_s=''):
3092 3092 """Manage IPython's bookmark system.
3093 3093
3094 3094 %bookmark <name> - set bookmark to current dir
3095 3095 %bookmark <name> <dir> - set bookmark to <dir>
3096 3096 %bookmark -l - list all bookmarks
3097 3097 %bookmark -d <name> - remove bookmark
3098 3098 %bookmark -r - remove all bookmarks
3099 3099
3100 3100 You can later on access a bookmarked folder with:
3101 3101 %cd -b <name>
3102 3102 or simply '%cd <name>' if there is no directory called <name> AND
3103 3103 there is such a bookmark defined.
3104 3104
3105 3105 Your bookmarks persist through IPython sessions, but they are
3106 3106 associated with each profile."""
3107 3107
3108 3108 opts,args = self.parse_options(parameter_s,'drl',mode='list')
3109 3109 if len(args) > 2:
3110 3110 raise UsageError("%bookmark: too many arguments")
3111 3111
3112 3112 bkms = self.db.get('bookmarks',{})
3113 3113
3114 3114 if opts.has_key('d'):
3115 3115 try:
3116 3116 todel = args[0]
3117 3117 except IndexError:
3118 3118 raise UsageError(
3119 3119 "%bookmark -d: must provide a bookmark to delete")
3120 3120 else:
3121 3121 try:
3122 3122 del bkms[todel]
3123 3123 except KeyError:
3124 3124 raise UsageError(
3125 3125 "%%bookmark -d: Can't delete bookmark '%s'" % todel)
3126 3126
3127 3127 elif opts.has_key('r'):
3128 3128 bkms = {}
3129 3129 elif opts.has_key('l'):
3130 3130 bks = bkms.keys()
3131 3131 bks.sort()
3132 3132 if bks:
3133 3133 size = max(map(len,bks))
3134 3134 else:
3135 3135 size = 0
3136 3136 fmt = '%-'+str(size)+'s -> %s'
3137 3137 print 'Current bookmarks:'
3138 3138 for bk in bks:
3139 3139 print fmt % (bk,bkms[bk])
3140 3140 else:
3141 3141 if not args:
3142 3142 raise UsageError("%bookmark: You must specify the bookmark name")
3143 3143 elif len(args)==1:
3144 3144 bkms[args[0]] = os.getcwd()
3145 3145 elif len(args)==2:
3146 3146 bkms[args[0]] = args[1]
3147 3147 self.db['bookmarks'] = bkms
3148 3148
3149 3149 def magic_pycat(self, parameter_s=''):
3150 3150 """Show a syntax-highlighted file through a pager.
3151 3151
3152 3152 This magic is similar to the cat utility, but it will assume the file
3153 3153 to be Python source and will show it with syntax highlighting. """
3154 3154
3155 3155 try:
3156 3156 filename = get_py_filename(parameter_s)
3157 3157 cont = file_read(filename)
3158 3158 except IOError:
3159 3159 try:
3160 3160 cont = eval(parameter_s,self.user_ns)
3161 3161 except NameError:
3162 3162 cont = None
3163 3163 if cont is None:
3164 3164 print "Error: no such file or variable"
3165 3165 return
3166 3166
3167 3167 page.page(self.shell.pycolorize(cont))
3168 3168
3169 3169 def _rerun_pasted(self):
3170 3170 """ Rerun a previously pasted command.
3171 3171 """
3172 3172 b = self.user_ns.get('pasted_block', None)
3173 3173 if b is None:
3174 3174 raise UsageError('No previous pasted block available')
3175 3175 print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
3176 3176 exec b in self.user_ns
3177 3177
3178 3178 def _get_pasted_lines(self, sentinel):
3179 3179 """ Yield pasted lines until the user enters the given sentinel value.
3180 3180 """
3181 3181 from IPython.core import interactiveshell
3182 3182 print "Pasting code; enter '%s' alone on the line to stop." % sentinel
3183 3183 while True:
3184 3184 l = interactiveshell.raw_input_original(':')
3185 3185 if l == sentinel:
3186 3186 return
3187 3187 else:
3188 3188 yield l
3189 3189
3190 3190 def _strip_pasted_lines_for_code(self, raw_lines):
3191 3191 """ Strip non-code parts of a sequence of lines to return a block of
3192 3192 code.
3193 3193 """
3194 3194 # Regular expressions that declare text we strip from the input:
3195 3195 strip_re = [r'^\s*In \[\d+\]:', # IPython input prompt
3196 3196 r'^\s*(\s?>)+', # Python input prompt
3197 3197 r'^\s*\.{3,}', # Continuation prompts
3198 3198 r'^\++',
3199 3199 ]
3200 3200
3201 3201 strip_from_start = map(re.compile,strip_re)
3202 3202
3203 3203 lines = []
3204 3204 for l in raw_lines:
3205 3205 for pat in strip_from_start:
3206 3206 l = pat.sub('',l)
3207 3207 lines.append(l)
3208 3208
3209 3209 block = "\n".join(lines) + '\n'
3210 3210 #print "block:\n",block
3211 3211 return block
3212 3212
3213 3213 def _execute_block(self, block, par):
3214 3214 """ Execute a block, or store it in a variable, per the user's request.
3215 3215 """
3216 3216 if not par:
3217 3217 b = textwrap.dedent(block)
3218 3218 self.user_ns['pasted_block'] = b
3219 3219 exec b in self.user_ns
3220 3220 else:
3221 3221 self.user_ns[par] = SList(block.splitlines())
3222 3222 print "Block assigned to '%s'" % par
3223 3223
3224 3224 def magic_quickref(self,arg):
3225 3225 """ Show a quick reference sheet """
3226 3226 import IPython.core.usage
3227 3227 qr = IPython.core.usage.quick_reference + self.magic_magic('-brief')
3228 3228
3229 3229 page.page(qr)
3230 3230
3231 3231 def magic_doctest_mode(self,parameter_s=''):
3232 3232 """Toggle doctest mode on and off.
3233 3233
3234 3234 This mode is intended to make IPython behave as much as possible like a
3235 3235 plain Python shell, from the perspective of how its prompts, exceptions
3236 3236 and output look. This makes it easy to copy and paste parts of a
3237 3237 session into doctests. It does so by:
3238 3238
3239 3239 - Changing the prompts to the classic ``>>>`` ones.
3240 3240 - Changing the exception reporting mode to 'Plain'.
3241 3241 - Disabling pretty-printing of output.
3242 3242
3243 3243 Note that IPython also supports the pasting of code snippets that have
3244 3244 leading '>>>' and '...' prompts in them. This means that you can paste
3245 3245 doctests from files or docstrings (even if they have leading
3246 3246 whitespace), and the code will execute correctly. You can then use
3247 3247 '%history -t' to see the translated history; this will give you the
3248 3248 input after removal of all the leading prompts and whitespace, which
3249 3249 can be pasted back into an editor.
3250 3250
3251 3251 With these features, you can switch into this mode easily whenever you
3252 3252 need to do testing and changes to doctests, without having to leave
3253 3253 your existing IPython session.
3254 3254 """
3255 3255
3256 3256 from IPython.utils.ipstruct import Struct
3257 3257
3258 3258 # Shorthands
3259 3259 shell = self.shell
3260 3260 oc = shell.displayhook
3261 3261 meta = shell.meta
3262 3262 disp_formatter = self.shell.display_formatter
3263 3263 ptformatter = disp_formatter.formatters['text/plain']
3264 3264 # dstore is a data store kept in the instance metadata bag to track any
3265 3265 # changes we make, so we can undo them later.
3266 3266 dstore = meta.setdefault('doctest_mode',Struct())
3267 3267 save_dstore = dstore.setdefault
3268 3268
3269 3269 # save a few values we'll need to recover later
3270 3270 mode = save_dstore('mode',False)
3271 3271 save_dstore('rc_pprint',ptformatter.pprint)
3272 3272 save_dstore('xmode',shell.InteractiveTB.mode)
3273 3273 save_dstore('rc_separate_out',shell.separate_out)
3274 3274 save_dstore('rc_separate_out2',shell.separate_out2)
3275 3275 save_dstore('rc_prompts_pad_left',shell.prompts_pad_left)
3276 3276 save_dstore('rc_separate_in',shell.separate_in)
3277 3277 save_dstore('rc_plain_text_only',disp_formatter.plain_text_only)
3278 3278
3279 3279 if mode == False:
3280 3280 # turn on
3281 3281 oc.prompt1.p_template = '>>> '
3282 3282 oc.prompt2.p_template = '... '
3283 3283 oc.prompt_out.p_template = ''
3284 3284
3285 3285 # Prompt separators like plain python
3286 3286 oc.input_sep = oc.prompt1.sep = ''
3287 3287 oc.output_sep = ''
3288 3288 oc.output_sep2 = ''
3289 3289
3290 3290 oc.prompt1.pad_left = oc.prompt2.pad_left = \
3291 3291 oc.prompt_out.pad_left = False
3292 3292
3293 3293 ptformatter.pprint = False
3294 3294 disp_formatter.plain_text_only = True
3295 3295
3296 3296 shell.magic_xmode('Plain')
3297 3297 else:
3298 3298 # turn off
3299 3299 oc.prompt1.p_template = shell.prompt_in1
3300 3300 oc.prompt2.p_template = shell.prompt_in2
3301 3301 oc.prompt_out.p_template = shell.prompt_out
3302 3302
3303 3303 oc.input_sep = oc.prompt1.sep = dstore.rc_separate_in
3304 3304
3305 3305 oc.output_sep = dstore.rc_separate_out
3306 3306 oc.output_sep2 = dstore.rc_separate_out2
3307 3307
3308 3308 oc.prompt1.pad_left = oc.prompt2.pad_left = \
3309 3309 oc.prompt_out.pad_left = dstore.rc_prompts_pad_left
3310 3310
3311 3311 ptformatter.pprint = dstore.rc_pprint
3312 3312 disp_formatter.plain_text_only = dstore.rc_plain_text_only
3313 3313
3314 3314 shell.magic_xmode(dstore.xmode)
3315 3315
3316 3316 # Store new mode and inform
3317 3317 dstore.mode = bool(1-int(mode))
3318 3318 mode_label = ['OFF','ON'][dstore.mode]
3319 3319 print 'Doctest mode is:', mode_label
3320 3320
3321 3321 def magic_gui(self, parameter_s=''):
3322 3322 """Enable or disable IPython GUI event loop integration.
3323 3323
3324 3324 %gui [GUINAME]
3325 3325
3326 3326 This magic replaces IPython's threaded shells that were activated
3327 3327 using the (pylab/wthread/etc.) command line flags. GUI toolkits
3328 3328 can now be enabled, disabled and changed at runtime and keyboard
3329 3329 interrupts should work without any problems. The following toolkits
3330 3330 are supported: wxPython, PyQt4, PyGTK, and Tk::
3331 3331
3332 3332 %gui wx # enable wxPython event loop integration
3333 3333 %gui qt4|qt # enable PyQt4 event loop integration
3334 3334 %gui gtk # enable PyGTK event loop integration
3335 3335 %gui tk # enable Tk event loop integration
3336 3336 %gui # disable all event loop integration
3337 3337
3338 3338 WARNING: after any of these has been called you can simply create
3339 3339 an application object, but DO NOT start the event loop yourself, as
3340 3340 we have already handled that.
3341 3341 """
3342 3342 from IPython.lib.inputhook import enable_gui
3343 3343 opts, arg = self.parse_options(parameter_s, '')
3344 3344 if arg=='': arg = None
3345 3345 return enable_gui(arg)
3346 3346
3347 3347 def magic_load_ext(self, module_str):
3348 3348 """Load an IPython extension by its module name."""
3349 3349 return self.extension_manager.load_extension(module_str)
3350 3350
3351 3351 def magic_unload_ext(self, module_str):
3352 3352 """Unload an IPython extension by its module name."""
3353 3353 self.extension_manager.unload_extension(module_str)
3354 3354
3355 3355 def magic_reload_ext(self, module_str):
3356 3356 """Reload an IPython extension by its module name."""
3357 3357 self.extension_manager.reload_extension(module_str)
3358 3358
3359 3359 @skip_doctest
3360 3360 def magic_install_profiles(self, s):
3361 3361 """Install the default IPython profiles into the .ipython dir.
3362 3362
3363 3363 If the default profiles have already been installed, they will not
3364 3364 be overwritten. You can force overwriting them by using the ``-o``
3365 3365 option::
3366 3366
3367 3367 In [1]: %install_profiles -o
3368 3368 """
3369 3369 if '-o' in s:
3370 3370 overwrite = True
3371 3371 else:
3372 3372 overwrite = False
3373 3373 from IPython.config import profile
3374 3374 profile_dir = os.path.dirname(profile.__file__)
3375 3375 ipython_dir = self.ipython_dir
3376 3376 print "Installing profiles to: %s [overwrite=%s]"%(ipython_dir,overwrite)
3377 3377 for src in os.listdir(profile_dir):
3378 3378 if src.startswith('profile_'):
3379 3379 name = src.replace('profile_', '')
3380 3380 print " %s"%name
3381 3381 pd = ProfileDir.create_profile_dir_by_name(ipython_dir, name)
3382 3382 pd.copy_config_file('ipython_config.py', path=src,
3383 3383 overwrite=overwrite)
3384 3384
3385 3385 @skip_doctest
3386 3386 def magic_install_default_config(self, s):
3387 3387 """Install IPython's default config file into the .ipython dir.
3388 3388
3389 3389 If the default config file (:file:`ipython_config.py`) is already
3390 3390 installed, it will not be overwritten. You can force overwriting
3391 3391 by using the ``-o`` option::
3392 3392
3393 3393 In [1]: %install_default_config
3394 3394 """
3395 3395 if '-o' in s:
3396 3396 overwrite = True
3397 3397 else:
3398 3398 overwrite = False
3399 3399 pd = self.shell.profile_dir
3400 3400 print "Installing default config file in: %s" % pd.location
3401 3401 pd.copy_config_file('ipython_config.py', overwrite=overwrite)
3402 3402
3403 3403 # Pylab support: simple wrappers that activate pylab, load gui input
3404 3404 # handling and modify slightly %run
3405 3405
3406 3406 @skip_doctest
3407 3407 def _pylab_magic_run(self, parameter_s=''):
3408 3408 Magic.magic_run(self, parameter_s,
3409 3409 runner=mpl_runner(self.shell.safe_execfile))
3410 3410
3411 3411 _pylab_magic_run.__doc__ = magic_run.__doc__
3412 3412
3413 3413 @skip_doctest
3414 3414 def magic_pylab(self, s):
3415 3415 """Load numpy and matplotlib to work interactively.
3416 3416
3417 3417 %pylab [GUINAME]
3418 3418
3419 3419 This function lets you activate pylab (matplotlib, numpy and
3420 3420 interactive support) at any point during an IPython session.
3421 3421
3422 3422 It will import at the top level numpy as np, pyplot as plt, matplotlib,
3423 3423 pylab and mlab, as well as all names from numpy and pylab.
3424 3424
3425 3425 Parameters
3426 3426 ----------
3427 3427 guiname : optional
3428 3428 One of the valid arguments to the %gui magic ('qt', 'wx', 'gtk', 'osx' or
3429 3429 'tk'). If given, the corresponding Matplotlib backend is used,
3430 3430 otherwise matplotlib's default (which you can override in your
3431 3431 matplotlib config file) is used.
3432 3432
3433 3433 Examples
3434 3434 --------
3435 3435 In this case, where the MPL default is TkAgg:
3436 3436 In [2]: %pylab
3437 3437
3438 3438 Welcome to pylab, a matplotlib-based Python environment.
3439 3439 Backend in use: TkAgg
3440 3440 For more information, type 'help(pylab)'.
3441 3441
3442 3442 But you can explicitly request a different backend:
3443 3443 In [3]: %pylab qt
3444 3444
3445 3445 Welcome to pylab, a matplotlib-based Python environment.
3446 3446 Backend in use: Qt4Agg
3447 3447 For more information, type 'help(pylab)'.
3448 3448 """
3449 3449 self.shell.enable_pylab(s)
3450 3450
3451 3451 def magic_tb(self, s):
3452 3452 """Print the last traceback with the currently active exception mode.
3453 3453
3454 3454 See %xmode for changing exception reporting modes."""
3455 3455 self.shell.showtraceback()
3456 3456
3457 3457 @skip_doctest
3458 3458 def magic_precision(self, s=''):
3459 3459 """Set floating point precision for pretty printing.
3460 3460
3461 3461 Can set either integer precision or a format string.
3462 3462
3463 3463 If numpy has been imported and precision is an int,
3464 3464 numpy display precision will also be set, via ``numpy.set_printoptions``.
3465 3465
3466 3466 If no argument is given, defaults will be restored.
3467 3467
3468 3468 Examples
3469 3469 --------
3470 3470 ::
3471 3471
3472 3472 In [1]: from math import pi
3473 3473
3474 3474 In [2]: %precision 3
3475 3475 Out[2]: '%.3f'
3476 3476
3477 3477 In [3]: pi
3478 3478 Out[3]: 3.142
3479 3479
3480 3480 In [4]: %precision %i
3481 3481 Out[4]: '%i'
3482 3482
3483 3483 In [5]: pi
3484 3484 Out[5]: 3
3485 3485
3486 3486 In [6]: %precision %e
3487 3487 Out[6]: '%e'
3488 3488
3489 3489 In [7]: pi**10
3490 3490 Out[7]: 9.364805e+04
3491 3491
3492 3492 In [8]: %precision
3493 3493 Out[8]: '%r'
3494 3494
3495 3495 In [9]: pi**10
3496 3496 Out[9]: 93648.047476082982
3497 3497
3498 3498 """
3499 3499
3500 3500 ptformatter = self.shell.display_formatter.formatters['text/plain']
3501 3501 ptformatter.float_precision = s
3502 3502 return ptformatter.float_format
3503 3503
3504 3504 # end Magic
@@ -1,244 +1,244 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 A mixin for :class:`~IPython.core.newapplication.Application` classes that
4 A mixin for :class:`~IPython.core.application.Application` classes that
5 5 launch InteractiveShell instances, load extensions, etc.
6 6
7 7 Authors
8 8 -------
9 9
10 10 * Min Ragan-Kelley
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 from __future__ import absolute_import
25 25
26 26 import os
27 27 import sys
28 28
29 29 from IPython.config.application import boolean_flag
30 30 from IPython.config.configurable import Configurable
31 31 from IPython.config.loader import Config
32 32 from IPython.utils.path import filefind
33 33 from IPython.utils.traitlets import Unicode, Instance, List
34 34
35 35 #-----------------------------------------------------------------------------
36 36 # Aliases and Flags
37 37 #-----------------------------------------------------------------------------
38 38
39 39 shell_flags = {}
40 40
41 41 addflag = lambda *args: shell_flags.update(boolean_flag(*args))
42 42 addflag('autoindent', 'InteractiveShell.autoindent',
43 43 'Turn on autoindenting.', 'Turn off autoindenting.'
44 44 )
45 45 addflag('automagic', 'InteractiveShell.automagic',
46 46 """Turn on the auto calling of magic commands. Type %%magic at the
47 47 IPython prompt for more information.""",
48 48 'Turn off the auto calling of magic commands.'
49 49 )
50 50 addflag('pdb', 'InteractiveShell.pdb',
51 51 "Enable auto calling the pdb debugger after every exception.",
52 52 "Disable auto calling the pdb debugger after every exception."
53 53 )
54 54 addflag('pprint', 'PlainTextFormatter.pprint',
55 55 "Enable auto pretty printing of results.",
56 56 "Disable auto auto pretty printing of results."
57 57 )
58 58 addflag('color-info', 'InteractiveShell.color_info',
59 59 """IPython can display information about objects via a set of func-
60 60 tions, and optionally can use colors for this, syntax highlighting
61 61 source code and various other elements. However, because this
62 62 information is passed through a pager (like 'less') and many pagers get
63 63 confused with color codes, this option is off by default. You can test
64 64 it and turn it on permanently in your ipython_config.py file if it
65 65 works for you. Test it and turn it on permanently if it works with
66 66 your system. The magic function %%color_info allows you to toggle this
67 67 interactively for testing.""",
68 68 "Disable using colors for info related things."
69 69 )
70 70 addflag('deep-reload', 'InteractiveShell.deep_reload',
71 71 """Enable deep (recursive) reloading by default. IPython can use the
72 72 deep_reload module which reloads changes in modules recursively (it
73 73 replaces the reload() function, so you don't need to change anything to
74 74 use it). deep_reload() forces a full reload of modules whose code may
75 75 have changed, which the default reload() function does not. When
76 76 deep_reload is off, IPython will use the normal reload(), but
77 77 deep_reload will still be available as dreload(). This feature is off
78 78 by default [which means that you have both normal reload() and
79 79 dreload()].""",
80 80 "Disable deep (recursive) reloading by default."
81 81 )
82 82 nosep_config = Config()
83 83 nosep_config.InteractiveShell.separate_in = ''
84 84 nosep_config.InteractiveShell.separate_out = ''
85 85 nosep_config.InteractiveShell.separate_out2 = ''
86 86
87 87 shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.")
88 88
89 89
90 90 # it's possible we don't want short aliases for *all* of these:
91 91 shell_aliases = dict(
92 92 autocall='InteractiveShell.autocall',
93 93 cache_size='InteractiveShell.cache_size',
94 94 colors='InteractiveShell.colors',
95 95 logfile='InteractiveShell.logfile',
96 96 log_append='InteractiveShell.logappend',
97 97 c='InteractiveShellApp.code_to_run',
98 98 ext='InteractiveShellApp.extra_extension',
99 99 )
100 100
101 101 #-----------------------------------------------------------------------------
102 102 # Main classes and functions
103 103 #-----------------------------------------------------------------------------
104 104
105 105 class InteractiveShellApp(Configurable):
106 106 """A Mixin for applications that start InteractiveShell instances.
107 107
108 108 Provides configurables for loading extensions and executing files
109 109 as part of configuring a Shell environment.
110 110
111 111 Provides init_extensions() and init_code() methods, to be called
112 112 after init_shell(), which must be implemented by subclasses.
113 113 """
114 114 extensions = List(Unicode, config=True,
115 115 help="A list of dotted module names of IPython extensions to load."
116 116 )
117 117 extra_extension = Unicode('', config=True,
118 118 help="dotted module name of an IPython extension to load."
119 119 )
120 120 def _extra_extension_changed(self, name, old, new):
121 121 if new:
122 122 # add to self.extensions
123 123 self.extensions.append(new)
124 124
125 125 exec_files = List(Unicode, config=True,
126 126 help="""List of files to run at IPython startup."""
127 127 )
128 128 file_to_run = Unicode('', config=True,
129 129 help="""A file to be run""")
130 130
131 131 exec_lines = List(Unicode, config=True,
132 132 help="""lines of code to run at IPython startup."""
133 133 )
134 134 code_to_run = Unicode('', config=True,
135 135 help="Execute the given command string."
136 136 )
137 137 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
138 138
139 139 def init_shell(self):
140 140 raise NotImplementedError("Override in subclasses")
141 141
142 142 def init_extensions(self):
143 143 """Load all IPython extensions in IPythonApp.extensions.
144 144
145 145 This uses the :meth:`ExtensionManager.load_extensions` to load all
146 146 the extensions listed in ``self.extensions``.
147 147 """
148 148 if not self.extensions:
149 149 return
150 150 try:
151 151 self.log.debug("Loading IPython extensions...")
152 152 extensions = self.extensions
153 153 for ext in extensions:
154 154 try:
155 155 self.log.info("Loading IPython extension: %s" % ext)
156 156 self.shell.extension_manager.load_extension(ext)
157 157 except:
158 158 self.log.warn("Error in loading extension: %s" % ext)
159 159 self.shell.showtraceback()
160 160 except:
161 161 self.log.warn("Unknown error in loading extensions:")
162 162 self.shell.showtraceback()
163 163
164 164 def init_code(self):
165 165 """run the pre-flight code, specified via exec_lines"""
166 166 self._run_exec_lines()
167 167 self._run_exec_files()
168 168 self._run_cmd_line_code()
169 169
170 170 def _run_exec_lines(self):
171 171 """Run lines of code in IPythonApp.exec_lines in the user's namespace."""
172 172 if not self.exec_lines:
173 173 return
174 174 try:
175 175 self.log.debug("Running code from IPythonApp.exec_lines...")
176 176 for line in self.exec_lines:
177 177 try:
178 178 self.log.info("Running code in user namespace: %s" %
179 179 line)
180 180 self.shell.run_cell(line, store_history=False)
181 181 except:
182 182 self.log.warn("Error in executing line in user "
183 183 "namespace: %s" % line)
184 184 self.shell.showtraceback()
185 185 except:
186 186 self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
187 187 self.shell.showtraceback()
188 188
189 189 def _exec_file(self, fname):
190 190 full_filename = filefind(fname, [u'.', self.ipython_dir])
191 191 if os.path.isfile(full_filename):
192 192 if full_filename.endswith(u'.py'):
193 193 self.log.info("Running file in user namespace: %s" %
194 194 full_filename)
195 195 # Ensure that __file__ is always defined to match Python behavior
196 196 self.shell.user_ns['__file__'] = fname
197 197 try:
198 198 self.shell.safe_execfile(full_filename, self.shell.user_ns)
199 199 finally:
200 200 del self.shell.user_ns['__file__']
201 201 elif full_filename.endswith('.ipy'):
202 202 self.log.info("Running file in user namespace: %s" %
203 203 full_filename)
204 204 self.shell.safe_execfile_ipy(full_filename)
205 205 else:
206 206 self.log.warn("File does not have a .py or .ipy extension: <%s>"
207 207 % full_filename)
208 208
209 209 def _run_exec_files(self):
210 210 """Run files from IPythonApp.exec_files"""
211 211 if not self.exec_files:
212 212 return
213 213
214 214 self.log.debug("Running files in IPythonApp.exec_files...")
215 215 try:
216 216 for fname in self.exec_files:
217 217 self._exec_file(fname)
218 218 except:
219 219 self.log.warn("Unknown error in handling IPythonApp.exec_files:")
220 220 self.shell.showtraceback()
221 221
222 222 def _run_cmd_line_code(self):
223 223 """Run code or file specified at the command-line"""
224 224 if self.code_to_run:
225 225 line = self.code_to_run
226 226 try:
227 227 self.log.info("Running code given at command line (c=): %s" %
228 228 line)
229 229 self.shell.run_cell(line, store_history=False)
230 230 except:
231 231 self.log.warn("Error in executing line in user namespace: %s" %
232 232 line)
233 233 self.shell.showtraceback()
234 234
235 235 # Like Python itself, ignore the second if the first of these is present
236 236 elif self.file_to_run:
237 237 fname = self.file_to_run
238 238 try:
239 239 self._exec_file(fname)
240 240 except:
241 241 self.log.warn("Error in executing file in user namespace: %s" %
242 242 fname)
243 243 self.shell.showtraceback()
244 244
@@ -1,71 +1,49 b''
1 1 # coding: utf-8
2 2 """Tests for IPython.core.application"""
3 3
4 4 import os
5 5 import tempfile
6 6
7 from IPython.core.application import Application
7 from IPython.core.application import BaseIPythonApplication
8 8 from IPython.testing import decorators as testdec
9 9
10 10 @testdec.onlyif_unicode_paths
11 11 def test_unicode_cwd():
12 12 """Check that IPython starts with non-ascii characters in the path."""
13 13 wd = tempfile.mkdtemp(suffix=u"€")
14 14
15 15 old_wd = os.getcwdu()
16 16 os.chdir(wd)
17 17 #raise Exception(repr(os.getcwd()))
18 18 try:
19 app = Application()
19 app = BaseIPythonApplication()
20 20 # The lines below are copied from Application.initialize()
21 app.create_default_config()
22 app.log_default_config()
23 app.set_default_config_log_level()
24
25 # Find resources needed for filesystem access, using information from
26 # the above two
27 app.find_ipython_dir()
28 app.find_resources()
29 app.find_config_file_name()
30 app.find_config_file_paths()
31
32 # File-based config
33 app.pre_load_file_config()
34 app.load_file_config(suppress_errors=False)
21 app.init_profile_dir()
22 app.init_config_files()
23 app.load_config_file(suppress_errors=False)
35 24 finally:
36 25 os.chdir(old_wd)
37 26
38 27 @testdec.onlyif_unicode_paths
39 28 def test_unicode_ipdir():
40 29 """Check that IPython starts with non-ascii characters in the IP dir."""
41 30 ipdir = tempfile.mkdtemp(suffix=u"€")
42 31
43 32 # Create the config file, so it tries to load it.
44 33 with open(os.path.join(ipdir, 'ipython_config.py'), "w") as f:
45 34 pass
46 35
47 36 old_ipdir1 = os.environ.pop("IPYTHONDIR", None)
48 37 old_ipdir2 = os.environ.pop("IPYTHON_DIR", None)
49 38 os.environ["IPYTHONDIR"] = ipdir.encode("utf-8")
50 39 try:
51 app = Application()
40 app = BaseIPythonApplication()
52 41 # The lines below are copied from Application.initialize()
53 app.create_default_config()
54 app.log_default_config()
55 app.set_default_config_log_level()
56
57 # Find resources needed for filesystem access, using information from
58 # the above two
59 app.find_ipython_dir()
60 app.find_resources()
61 app.find_config_file_name()
62 app.find_config_file_paths()
63
64 # File-based config
65 app.pre_load_file_config()
66 app.load_file_config(suppress_errors=False)
42 app.init_profile_dir()
43 app.init_config_files()
44 app.load_config_file(suppress_errors=False)
67 45 finally:
68 46 if old_ipdir1:
69 47 os.environ["IPYTHONDIR"] = old_ipdir1
70 48 if old_ipdir2:
71 49 os.environ["IPYTHONDIR"] = old_ipdir2
@@ -1,420 +1,420 b''
1 1 """ A minimal application using the Qt console-style IPython frontend.
2 2
3 3 This is not a complete console app, as subprocess will not be able to receive
4 4 input, there is no real readline support, among other limitations.
5 5
6 6 Authors:
7 7
8 8 * Evan Patterson
9 9 * Min RK
10 10 * Erik Tollerud
11 11 * Fernando Perez
12 12
13 13 """
14 14
15 15 #-----------------------------------------------------------------------------
16 16 # Imports
17 17 #-----------------------------------------------------------------------------
18 18
19 19 # stdlib imports
20 20 import os
21 21 import signal
22 22 import sys
23 23
24 24 # System library imports
25 25 from IPython.external.qt import QtGui
26 26 from pygments.styles import get_all_styles
27 27
28 28 # Local imports
29 29 from IPython.config.application import boolean_flag
30 from IPython.core.newapplication import ProfileDir, BaseIPythonApplication
30 from IPython.core.application import ProfileDir, BaseIPythonApplication
31 31 from IPython.frontend.qt.console.frontend_widget import FrontendWidget
32 32 from IPython.frontend.qt.console.ipython_widget import IPythonWidget
33 33 from IPython.frontend.qt.console.rich_ipython_widget import RichIPythonWidget
34 34 from IPython.frontend.qt.console import styles
35 35 from IPython.frontend.qt.kernelmanager import QtKernelManager
36 36 from IPython.utils.traitlets import (
37 37 Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any
38 38 )
39 39 from IPython.zmq.ipkernel import (
40 40 flags as ipkernel_flags,
41 41 aliases as ipkernel_aliases,
42 42 IPKernelApp
43 43 )
44 44 from IPython.zmq.session import Session
45 45 from IPython.zmq.zmqshell import ZMQInteractiveShell
46 46
47 47
48 48 #-----------------------------------------------------------------------------
49 49 # Network Constants
50 50 #-----------------------------------------------------------------------------
51 51
52 52 from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
53 53
54 54 #-----------------------------------------------------------------------------
55 55 # Classes
56 56 #-----------------------------------------------------------------------------
57 57
58 58 class MainWindow(QtGui.QMainWindow):
59 59
60 60 #---------------------------------------------------------------------------
61 61 # 'object' interface
62 62 #---------------------------------------------------------------------------
63 63
64 64 def __init__(self, app, frontend, existing=False, may_close=True,
65 65 confirm_exit=True):
66 66 """ Create a MainWindow for the specified FrontendWidget.
67 67
68 68 The app is passed as an argument to allow for different
69 69 closing behavior depending on whether we are the Kernel's parent.
70 70
71 71 If existing is True, then this Console does not own the Kernel.
72 72
73 73 If may_close is True, then this Console is permitted to close the kernel
74 74 """
75 75 super(MainWindow, self).__init__()
76 76 self._app = app
77 77 self._frontend = frontend
78 78 self._existing = existing
79 79 if existing:
80 80 self._may_close = may_close
81 81 else:
82 82 self._may_close = True
83 83 self._frontend.exit_requested.connect(self.close)
84 84 self._confirm_exit = confirm_exit
85 85 self.setCentralWidget(frontend)
86 86
87 87 #---------------------------------------------------------------------------
88 88 # QWidget interface
89 89 #---------------------------------------------------------------------------
90 90
91 91 def closeEvent(self, event):
92 92 """ Close the window and the kernel (if necessary).
93 93
94 94 This will prompt the user if they are finished with the kernel, and if
95 95 so, closes the kernel cleanly. Alternatively, if the exit magic is used,
96 96 it closes without prompt.
97 97 """
98 98 keepkernel = None #Use the prompt by default
99 99 if hasattr(self._frontend,'_keep_kernel_on_exit'): #set by exit magic
100 100 keepkernel = self._frontend._keep_kernel_on_exit
101 101
102 102 kernel_manager = self._frontend.kernel_manager
103 103
104 104 if keepkernel is None and not self._confirm_exit:
105 105 # don't prompt, just terminate the kernel if we own it
106 106 # or leave it alone if we don't
107 107 keepkernel = not self._existing
108 108
109 109 if keepkernel is None: #show prompt
110 110 if kernel_manager and kernel_manager.channels_running:
111 111 title = self.window().windowTitle()
112 112 cancel = QtGui.QMessageBox.Cancel
113 113 okay = QtGui.QMessageBox.Ok
114 114 if self._may_close:
115 115 msg = "You are closing this Console window."
116 116 info = "Would you like to quit the Kernel and all attached Consoles as well?"
117 117 justthis = QtGui.QPushButton("&No, just this Console", self)
118 118 justthis.setShortcut('N')
119 119 closeall = QtGui.QPushButton("&Yes, quit everything", self)
120 120 closeall.setShortcut('Y')
121 121 box = QtGui.QMessageBox(QtGui.QMessageBox.Question,
122 122 title, msg)
123 123 box.setInformativeText(info)
124 124 box.addButton(cancel)
125 125 box.addButton(justthis, QtGui.QMessageBox.NoRole)
126 126 box.addButton(closeall, QtGui.QMessageBox.YesRole)
127 127 box.setDefaultButton(closeall)
128 128 box.setEscapeButton(cancel)
129 129 reply = box.exec_()
130 130 if reply == 1: # close All
131 131 kernel_manager.shutdown_kernel()
132 132 #kernel_manager.stop_channels()
133 133 event.accept()
134 134 elif reply == 0: # close Console
135 135 if not self._existing:
136 136 # Have kernel: don't quit, just close the window
137 137 self._app.setQuitOnLastWindowClosed(False)
138 138 self.deleteLater()
139 139 event.accept()
140 140 else:
141 141 event.ignore()
142 142 else:
143 143 reply = QtGui.QMessageBox.question(self, title,
144 144 "Are you sure you want to close this Console?"+
145 145 "\nThe Kernel and other Consoles will remain active.",
146 146 okay|cancel,
147 147 defaultButton=okay
148 148 )
149 149 if reply == okay:
150 150 event.accept()
151 151 else:
152 152 event.ignore()
153 153 elif keepkernel: #close console but leave kernel running (no prompt)
154 154 if kernel_manager and kernel_manager.channels_running:
155 155 if not self._existing:
156 156 # I have the kernel: don't quit, just close the window
157 157 self._app.setQuitOnLastWindowClosed(False)
158 158 event.accept()
159 159 else: #close console and kernel (no prompt)
160 160 if kernel_manager and kernel_manager.channels_running:
161 161 kernel_manager.shutdown_kernel()
162 162 event.accept()
163 163
164 164 #-----------------------------------------------------------------------------
165 165 # Aliases and Flags
166 166 #-----------------------------------------------------------------------------
167 167
168 168 flags = dict(ipkernel_flags)
169 169
170 170 flags.update({
171 171 'existing' : ({'IPythonQtConsoleApp' : {'existing' : True}},
172 172 "Connect to an existing kernel."),
173 173 'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}},
174 174 "Use a pure Python kernel instead of an IPython kernel."),
175 175 'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}},
176 176 "Disable rich text support."),
177 177 })
178 178 flags.update(boolean_flag(
179 179 'gui-completion', 'ConsoleWidget.gui_completion',
180 180 "use a GUI widget for tab completion",
181 181 "use plaintext output for completion"
182 182 ))
183 183 flags.update(boolean_flag(
184 184 'confirm-exit', 'IPythonQtConsoleApp.confirm_exit',
185 185 """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
186 186 to force a direct exit without any confirmation.
187 187 """,
188 188 """Don't prompt the user when exiting. This will terminate the kernel
189 189 if it is owned by the frontend, and leave it alive if it is external.
190 190 """
191 191 ))
192 192 # the flags that are specific to the frontend
193 193 # these must be scrubbed before being passed to the kernel,
194 194 # or it will raise an error on unrecognized flags
195 195 qt_flags = ['existing', 'pure', 'plain', 'gui-completion', 'no-gui-completion',
196 196 'confirm-exit', 'no-confirm-exit']
197 197
198 198 aliases = dict(ipkernel_aliases)
199 199
200 200 aliases.update(dict(
201 201 hb = 'IPythonQtConsoleApp.hb_port',
202 202 shell = 'IPythonQtConsoleApp.shell_port',
203 203 iopub = 'IPythonQtConsoleApp.iopub_port',
204 204 stdin = 'IPythonQtConsoleApp.stdin_port',
205 205 ip = 'IPythonQtConsoleApp.ip',
206 206
207 207 plain = 'IPythonQtConsoleApp.plain',
208 208 pure = 'IPythonQtConsoleApp.pure',
209 209 gui_completion = 'ConsoleWidget.gui_completion',
210 210 style = 'IPythonWidget.syntax_style',
211 211 stylesheet = 'IPythonQtConsoleApp.stylesheet',
212 212 colors = 'ZMQInteractiveShell.colors',
213 213
214 214 editor = 'IPythonWidget.editor',
215 215 ))
216 216
217 217 #-----------------------------------------------------------------------------
218 218 # IPythonQtConsole
219 219 #-----------------------------------------------------------------------------
220 220 class IPythonQtConsoleApp(BaseIPythonApplication):
221 221 name = 'ipython-qtconsole'
222 222 default_config_file_name='ipython_config.py'
223 223
224 224 description = """
225 225 The IPython QtConsole.
226 226
227 227 This launches a Console-style application using Qt. It is not a full
228 228 console, in that launched terminal subprocesses will not.
229 229
230 230 The QtConsole supports various extra features beyond the
231 231
232 232 """
233 233
234 234 classes = [IPKernelApp, IPythonWidget, ZMQInteractiveShell, ProfileDir, Session]
235 235 flags = Dict(flags)
236 236 aliases = Dict(aliases)
237 237
238 238 kernel_argv = List(Unicode)
239 239
240 240 # connection info:
241 241 ip = Unicode(LOCALHOST, config=True,
242 242 help="""Set the kernel\'s IP address [default localhost].
243 243 If the IP address is something other than localhost, then
244 244 Consoles on other machines will be able to connect
245 245 to the Kernel, so be careful!"""
246 246 )
247 247 hb_port = Int(0, config=True,
248 248 help="set the heartbeat port [default: random]")
249 249 shell_port = Int(0, config=True,
250 250 help="set the shell (XREP) port [default: random]")
251 251 iopub_port = Int(0, config=True,
252 252 help="set the iopub (PUB) port [default: random]")
253 253 stdin_port = Int(0, config=True,
254 254 help="set the stdin (XREQ) port [default: random]")
255 255
256 256 existing = CBool(False, config=True,
257 257 help="Whether to connect to an already running Kernel.")
258 258
259 259 stylesheet = Unicode('', config=True,
260 260 help="path to a custom CSS stylesheet")
261 261
262 262 pure = CBool(False, config=True,
263 263 help="Use a pure Python kernel instead of an IPython kernel.")
264 264 plain = CBool(False, config=True,
265 265 help="Use a plaintext widget instead of rich text (plain can't print/save).")
266 266
267 267 def _pure_changed(self, name, old, new):
268 268 kind = 'plain' if self.plain else 'rich'
269 269 self.config.ConsoleWidget.kind = kind
270 270 if self.pure:
271 271 self.widget_factory = FrontendWidget
272 272 elif self.plain:
273 273 self.widget_factory = IPythonWidget
274 274 else:
275 275 self.widget_factory = RichIPythonWidget
276 276
277 277 _plain_changed = _pure_changed
278 278
279 279 confirm_exit = CBool(True, config=True,
280 280 help="""
281 281 Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
282 282 to force a direct exit without any confirmation.""",
283 283 )
284 284
285 285 # the factory for creating a widget
286 286 widget_factory = Any(RichIPythonWidget)
287 287
288 288 def parse_command_line(self, argv=None):
289 289 super(IPythonQtConsoleApp, self).parse_command_line(argv)
290 290 if argv is None:
291 291 argv = sys.argv[1:]
292 292
293 293 self.kernel_argv = list(argv) # copy
294 294
295 295 # scrub frontend-specific flags
296 296 for a in argv:
297 297 if a.startswith('--') and a[2:] in qt_flags:
298 298 self.kernel_argv.remove(a)
299 299
300 300 def init_kernel_manager(self):
301 301 # Don't let Qt or ZMQ swallow KeyboardInterupts.
302 302 signal.signal(signal.SIGINT, signal.SIG_DFL)
303 303
304 304 # Create a KernelManager and start a kernel.
305 305 self.kernel_manager = QtKernelManager(
306 306 shell_address=(self.ip, self.shell_port),
307 307 sub_address=(self.ip, self.iopub_port),
308 308 stdin_address=(self.ip, self.stdin_port),
309 309 hb_address=(self.ip, self.hb_port),
310 310 config=self.config
311 311 )
312 312 # start the kernel
313 313 if not self.existing:
314 314 kwargs = dict(ip=self.ip, ipython=not self.pure)
315 315 kwargs['extra_arguments'] = self.kernel_argv
316 316 self.kernel_manager.start_kernel(**kwargs)
317 317 self.kernel_manager.start_channels()
318 318
319 319
320 320 def init_qt_elements(self):
321 321 # Create the widget.
322 322 self.app = QtGui.QApplication([])
323 323 local_kernel = (not self.existing) or self.ip in LOCAL_IPS
324 324 self.widget = self.widget_factory(config=self.config,
325 325 local_kernel=local_kernel)
326 326 self.widget.kernel_manager = self.kernel_manager
327 327 self.window = MainWindow(self.app, self.widget, self.existing,
328 328 may_close=local_kernel,
329 329 confirm_exit=self.confirm_exit)
330 330 self.window.setWindowTitle('Python' if self.pure else 'IPython')
331 331
332 332 def init_colors(self):
333 333 """Configure the coloring of the widget"""
334 334 # Note: This will be dramatically simplified when colors
335 335 # are removed from the backend.
336 336
337 337 if self.pure:
338 338 # only IPythonWidget supports styling
339 339 return
340 340
341 341 # parse the colors arg down to current known labels
342 342 try:
343 343 colors = self.config.ZMQInteractiveShell.colors
344 344 except AttributeError:
345 345 colors = None
346 346 try:
347 347 style = self.config.IPythonWidget.colors
348 348 except AttributeError:
349 349 style = None
350 350
351 351 # find the value for colors:
352 352 if colors:
353 353 colors=colors.lower()
354 354 if colors in ('lightbg', 'light'):
355 355 colors='lightbg'
356 356 elif colors in ('dark', 'linux'):
357 357 colors='linux'
358 358 else:
359 359 colors='nocolor'
360 360 elif style:
361 361 if style=='bw':
362 362 colors='nocolor'
363 363 elif styles.dark_style(style):
364 364 colors='linux'
365 365 else:
366 366 colors='lightbg'
367 367 else:
368 368 colors=None
369 369
370 370 # Configure the style.
371 371 widget = self.widget
372 372 if style:
373 373 widget.style_sheet = styles.sheet_from_template(style, colors)
374 374 widget.syntax_style = style
375 375 widget._syntax_style_changed()
376 376 widget._style_sheet_changed()
377 377 elif colors:
378 378 # use a default style
379 379 widget.set_default_style(colors=colors)
380 380 else:
381 381 # this is redundant for now, but allows the widget's
382 382 # defaults to change
383 383 widget.set_default_style()
384 384
385 385 if self.stylesheet:
386 386 # we got an expicit stylesheet
387 387 if os.path.isfile(self.stylesheet):
388 388 with open(self.stylesheet) as f:
389 389 sheet = f.read()
390 390 widget.style_sheet = sheet
391 391 widget._style_sheet_changed()
392 392 else:
393 393 raise IOError("Stylesheet %r not found."%self.stylesheet)
394 394
395 395 def initialize(self, argv=None):
396 396 super(IPythonQtConsoleApp, self).initialize(argv)
397 397 self.init_kernel_manager()
398 398 self.init_qt_elements()
399 399 self.init_colors()
400 400
401 401 def start(self):
402 402
403 403 # draw the window
404 404 self.window.show()
405 405
406 406 # Start the application main loop.
407 407 self.app.exec_()
408 408
409 409 #-----------------------------------------------------------------------------
410 410 # Main entry point
411 411 #-----------------------------------------------------------------------------
412 412
413 413 def main():
414 414 app = IPythonQtConsoleApp()
415 415 app.initialize()
416 416 app.start()
417 417
418 418
419 419 if __name__ == '__main__':
420 420 main()
@@ -1,356 +1,356 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 The :class:`~IPython.core.newapplication.Application` object for the command
4 The :class:`~IPython.core.application.Application` object for the command
5 5 line :command:`ipython` program.
6 6
7 7 Authors
8 8 -------
9 9
10 10 * Brian Granger
11 11 * Fernando Perez
12 12 * Min Ragan-Kelley
13 13 """
14 14
15 15 #-----------------------------------------------------------------------------
16 16 # Copyright (C) 2008-2010 The IPython Development Team
17 17 #
18 18 # Distributed under the terms of the BSD License. The full license is in
19 19 # the file COPYING, distributed as part of this software.
20 20 #-----------------------------------------------------------------------------
21 21
22 22 #-----------------------------------------------------------------------------
23 23 # Imports
24 24 #-----------------------------------------------------------------------------
25 25
26 26 from __future__ import absolute_import
27 27
28 28 import logging
29 29 import os
30 30 import sys
31 31
32 32 from IPython.config.loader import (
33 33 Config, PyFileConfigLoader
34 34 )
35 35 from IPython.config.application import boolean_flag
36 36 from IPython.core import release
37 37 from IPython.core import usage
38 38 from IPython.core.crashhandler import CrashHandler
39 39 from IPython.core.formatters import PlainTextFormatter
40 from IPython.core.newapplication import (
40 from IPython.core.application import (
41 41 ProfileDir, BaseIPythonApplication, base_flags, base_aliases
42 42 )
43 43 from IPython.core.shellapp import (
44 44 InteractiveShellApp, shell_flags, shell_aliases
45 45 )
46 46 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
47 47 from IPython.lib import inputhook
48 48 from IPython.utils.path import get_ipython_dir, check_for_old_config
49 49 from IPython.utils.traitlets import (
50 50 Bool, Dict, CaselessStrEnum
51 51 )
52 52
53 53 #-----------------------------------------------------------------------------
54 54 # Globals, utilities and helpers
55 55 #-----------------------------------------------------------------------------
56 56
57 57 #: The default config file name for this application.
58 58 default_config_file_name = u'ipython_config.py'
59 59
60 60
61 61 #-----------------------------------------------------------------------------
62 62 # Crash handler for this application
63 63 #-----------------------------------------------------------------------------
64 64
65 65 _message_template = """\
66 66 Oops, $self.app_name crashed. We do our best to make it stable, but...
67 67
68 68 A crash report was automatically generated with the following information:
69 69 - A verbatim copy of the crash traceback.
70 70 - A copy of your input history during this session.
71 71 - Data on your current $self.app_name configuration.
72 72
73 73 It was left in the file named:
74 74 \t'$self.crash_report_fname'
75 75 If you can email this file to the developers, the information in it will help
76 76 them in understanding and correcting the problem.
77 77
78 78 You can mail it to: $self.contact_name at $self.contact_email
79 79 with the subject '$self.app_name Crash Report'.
80 80
81 81 If you want to do it now, the following command will work (under Unix):
82 82 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
83 83
84 84 To ensure accurate tracking of this issue, please file a report about it at:
85 85 $self.bug_tracker
86 86 """
87 87
88 88 class IPAppCrashHandler(CrashHandler):
89 89 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
90 90
91 91 message_template = _message_template
92 92
93 93 def __init__(self, app):
94 94 contact_name = release.authors['Fernando'][0]
95 95 contact_email = release.authors['Fernando'][1]
96 96 bug_tracker = 'http://github.com/ipython/ipython/issues'
97 97 super(IPAppCrashHandler,self).__init__(
98 98 app, contact_name, contact_email, bug_tracker
99 99 )
100 100
101 101 def make_report(self,traceback):
102 102 """Return a string containing a crash report."""
103 103
104 104 sec_sep = self.section_sep
105 105 # Start with parent report
106 106 report = [super(IPAppCrashHandler, self).make_report(traceback)]
107 107 # Add interactive-specific info we may have
108 108 rpt_add = report.append
109 109 try:
110 110 rpt_add(sec_sep+"History of session input:")
111 111 for line in self.app.shell.user_ns['_ih']:
112 112 rpt_add(line)
113 113 rpt_add('\n*** Last line of input (may not be in above history):\n')
114 114 rpt_add(self.app.shell._last_input_line+'\n')
115 115 except:
116 116 pass
117 117
118 118 return ''.join(report)
119 119
120 120 #-----------------------------------------------------------------------------
121 121 # Aliases and Flags
122 122 #-----------------------------------------------------------------------------
123 123 flags = dict(base_flags)
124 124 flags.update(shell_flags)
125 125 addflag = lambda *args: flags.update(boolean_flag(*args))
126 126 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
127 127 'Turn on auto editing of files with syntax errors.',
128 128 'Turn off auto editing of files with syntax errors.'
129 129 )
130 130 addflag('banner', 'TerminalIPythonApp.display_banner',
131 131 "Display a banner upon starting IPython.",
132 132 "Don't display a banner upon starting IPython."
133 133 )
134 134 addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
135 135 """Set to confirm when you try to exit IPython with an EOF (Control-D
136 136 in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
137 137 you can force a direct exit without any confirmation.""",
138 138 "Don't prompt the user when exiting."
139 139 )
140 140 addflag('term-title', 'TerminalInteractiveShell.term_title',
141 141 "Enable auto setting the terminal title.",
142 142 "Disable auto setting the terminal title."
143 143 )
144 144 classic_config = Config()
145 145 classic_config.InteractiveShell.cache_size = 0
146 146 classic_config.PlainTextFormatter.pprint = False
147 147 classic_config.InteractiveShell.prompt_in1 = '>>> '
148 148 classic_config.InteractiveShell.prompt_in2 = '... '
149 149 classic_config.InteractiveShell.prompt_out = ''
150 150 classic_config.InteractiveShell.separate_in = ''
151 151 classic_config.InteractiveShell.separate_out = ''
152 152 classic_config.InteractiveShell.separate_out2 = ''
153 153 classic_config.InteractiveShell.colors = 'NoColor'
154 154 classic_config.InteractiveShell.xmode = 'Plain'
155 155
156 156 flags['classic']=(
157 157 classic_config,
158 158 "Gives IPython a similar feel to the classic Python prompt."
159 159 )
160 160 # # log doesn't make so much sense this way anymore
161 161 # paa('--log','-l',
162 162 # action='store_true', dest='InteractiveShell.logstart',
163 163 # help="Start logging to the default log file (./ipython_log.py).")
164 164 #
165 165 # # quick is harder to implement
166 166 flags['quick']=(
167 167 {'TerminalIPythonApp' : {'quick' : True}},
168 168 "Enable quick startup with no config files."
169 169 )
170 170
171 171 flags['i'] = (
172 172 {'TerminalIPythonApp' : {'force_interact' : True}},
173 173 "If running code from the command line, become interactive afterwards."
174 174 )
175 175 flags['pylab'] = (
176 176 {'TerminalIPythonApp' : {'pylab' : 'auto'}},
177 177 """Pre-load matplotlib and numpy for interactive use with
178 178 the default matplotlib backend."""
179 179 )
180 180
181 181 aliases = dict(base_aliases)
182 182 aliases.update(shell_aliases)
183 183
184 184 # it's possible we don't want short aliases for *all* of these:
185 185 aliases.update(dict(
186 186 gui='TerminalIPythonApp.gui',
187 187 pylab='TerminalIPythonApp.pylab',
188 188 ))
189 189
190 190 #-----------------------------------------------------------------------------
191 191 # Main classes and functions
192 192 #-----------------------------------------------------------------------------
193 193
194 194 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
195 195 name = u'ipython'
196 196 description = usage.cl_usage
197 197 default_config_file_name = default_config_file_name
198 198 crash_handler_class = IPAppCrashHandler
199 199
200 200 flags = Dict(flags)
201 201 aliases = Dict(aliases)
202 202 classes = [InteractiveShellApp, TerminalInteractiveShell, ProfileDir, PlainTextFormatter]
203 203 subcommands = Dict(dict(
204 204 qtconsole=('IPython.frontend.qt.console.qtconsoleapp.IPythonQtConsoleApp',
205 205 """Launch the IPython Qt Console."""
206 206 )
207 207 ))
208 208
209 209 # *do* autocreate requested profile
210 210 auto_create=Bool(True)
211 211 copy_config_files=Bool(True)
212 212 # configurables
213 213 ignore_old_config=Bool(False, config=True,
214 214 help="Suppress warning messages about legacy config files"
215 215 )
216 216 quick = Bool(False, config=True,
217 217 help="""Start IPython quickly by skipping the loading of config files."""
218 218 )
219 219 def _quick_changed(self, name, old, new):
220 220 if new:
221 221 self.load_config_file = lambda *a, **kw: None
222 222 self.ignore_old_config=True
223 223
224 224 gui = CaselessStrEnum(('qt','wx','gtk'), config=True,
225 225 help="Enable GUI event loop integration ('qt', 'wx', 'gtk')."
226 226 )
227 227 pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'auto'],
228 228 config=True,
229 229 help="""Pre-load matplotlib and numpy for interactive use,
230 230 selecting a particular matplotlib backend and loop integration.
231 231 """
232 232 )
233 233 display_banner = Bool(True, config=True,
234 234 help="Whether to display a banner upon starting IPython."
235 235 )
236 236
237 237 # if there is code of files to run from the cmd line, don't interact
238 238 # unless the --i flag (App.force_interact) is true.
239 239 force_interact = Bool(False, config=True,
240 240 help="""If a command or file is given via the command-line,
241 241 e.g. 'ipython foo.py"""
242 242 )
243 243 def _force_interact_changed(self, name, old, new):
244 244 if new:
245 245 self.interact = True
246 246
247 247 def _file_to_run_changed(self, name, old, new):
248 248 if new and not self.force_interact:
249 249 self.interact = False
250 250 _code_to_run_changed = _file_to_run_changed
251 251
252 252 # internal, not-configurable
253 253 interact=Bool(True)
254 254
255 255
256 256 def initialize(self, argv=None):
257 257 """Do actions after construct, but before starting the app."""
258 258 super(TerminalIPythonApp, self).initialize(argv)
259 259 if self.subapp is not None:
260 260 # don't bother initializing further, starting subapp
261 261 return
262 262 if not self.ignore_old_config:
263 263 check_for_old_config(self.ipython_dir)
264 264 # print self.extra_args
265 265 if self.extra_args:
266 266 self.file_to_run = self.extra_args[0]
267 267 # create the shell
268 268 self.init_shell()
269 269 # and draw the banner
270 270 self.init_banner()
271 271 # Now a variety of things that happen after the banner is printed.
272 272 self.init_gui_pylab()
273 273 self.init_extensions()
274 274 self.init_code()
275 275
276 276 def init_shell(self):
277 277 """initialize the InteractiveShell instance"""
278 278 # I am a little hesitant to put these into InteractiveShell itself.
279 279 # But that might be the place for them
280 280 sys.path.insert(0, '')
281 281
282 282 # Create an InteractiveShell instance.
283 283 # shell.display_banner should always be False for the terminal
284 284 # based app, because we call shell.show_banner() by hand below
285 285 # so the banner shows *before* all extension loading stuff.
286 286 self.shell = TerminalInteractiveShell.instance(config=self.config,
287 287 display_banner=False, profile_dir=self.profile_dir,
288 288 ipython_dir=self.ipython_dir)
289 289
290 290 def init_banner(self):
291 291 """optionally display the banner"""
292 292 if self.display_banner and self.interact:
293 293 self.shell.show_banner()
294 294 # Make sure there is a space below the banner.
295 295 if self.log_level <= logging.INFO: print
296 296
297 297
298 298 def init_gui_pylab(self):
299 299 """Enable GUI event loop integration, taking pylab into account."""
300 300 gui = self.gui
301 301
302 302 # Using `pylab` will also require gui activation, though which toolkit
303 303 # to use may be chosen automatically based on mpl configuration.
304 304 if self.pylab:
305 305 activate = self.shell.enable_pylab
306 306 if self.pylab == 'auto':
307 307 gui = None
308 308 else:
309 309 gui = self.pylab
310 310 else:
311 311 # Enable only GUI integration, no pylab
312 312 activate = inputhook.enable_gui
313 313
314 314 if gui or self.pylab:
315 315 try:
316 316 self.log.info("Enabling GUI event loop integration, "
317 317 "toolkit=%s, pylab=%s" % (gui, self.pylab) )
318 318 activate(gui)
319 319 except:
320 320 self.log.warn("Error in enabling GUI event loop integration:")
321 321 self.shell.showtraceback()
322 322
323 323 def start(self):
324 324 if self.subapp is not None:
325 325 return self.subapp.start()
326 326 # perform any prexec steps:
327 327 if self.interact:
328 328 self.log.debug("Starting IPython's mainloop...")
329 329 self.shell.mainloop()
330 330 else:
331 331 self.log.debug("IPython not interactive...")
332 332
333 333
334 334 def load_default_config(ipython_dir=None):
335 335 """Load the default config file from the default ipython_dir.
336 336
337 337 This is useful for embedded shells.
338 338 """
339 339 if ipython_dir is None:
340 340 ipython_dir = get_ipython_dir()
341 341 profile_dir = os.path.join(ipython_dir, 'profile_default')
342 342 cl = PyFileConfigLoader(default_config_file_name, profile_dir)
343 343 config = cl.load_config()
344 344 return config
345 345
346 346
347 347 def launch_new_instance():
348 348 """Create and run a full blown IPython instance"""
349 349 app = TerminalIPythonApp.instance()
350 350 app.initialize()
351 351 app.start()
352 352
353 353
354 354 if __name__ == '__main__':
355 355 launch_new_instance()
356 356
@@ -1,263 +1,263 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The Base Application class for IPython.parallel apps
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * Min RK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 from __future__ import with_statement
25 25
26 26 import os
27 27 import logging
28 28 import re
29 29 import sys
30 30
31 31 from subprocess import Popen, PIPE
32 32
33 33 from IPython.core import release
34 34 from IPython.core.crashhandler import CrashHandler
35 from IPython.core.newapplication import (
35 from IPython.core.application import (
36 36 BaseIPythonApplication,
37 37 base_aliases as base_ip_aliases,
38 38 base_flags as base_ip_flags
39 39 )
40 40 from IPython.utils.path import expand_path
41 41
42 42 from IPython.utils.traitlets import Unicode, Bool, Instance, Dict, List
43 43
44 44 #-----------------------------------------------------------------------------
45 45 # Module errors
46 46 #-----------------------------------------------------------------------------
47 47
48 48 class PIDFileError(Exception):
49 49 pass
50 50
51 51
52 52 #-----------------------------------------------------------------------------
53 53 # Crash handler for this application
54 54 #-----------------------------------------------------------------------------
55 55
56 56
57 57 _message_template = """\
58 58 Oops, $self.app_name crashed. We do our best to make it stable, but...
59 59
60 60 A crash report was automatically generated with the following information:
61 61 - A verbatim copy of the crash traceback.
62 62 - Data on your current $self.app_name configuration.
63 63
64 64 It was left in the file named:
65 65 \t'$self.crash_report_fname'
66 66 If you can email this file to the developers, the information in it will help
67 67 them in understanding and correcting the problem.
68 68
69 69 You can mail it to: $self.contact_name at $self.contact_email
70 70 with the subject '$self.app_name Crash Report'.
71 71
72 72 If you want to do it now, the following command will work (under Unix):
73 73 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
74 74
75 75 To ensure accurate tracking of this issue, please file a report about it at:
76 76 $self.bug_tracker
77 77 """
78 78
79 79 class ParallelCrashHandler(CrashHandler):
80 80 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
81 81
82 82 message_template = _message_template
83 83
84 84 def __init__(self, app):
85 85 contact_name = release.authors['Min'][0]
86 86 contact_email = release.authors['Min'][1]
87 87 bug_tracker = 'http://github.com/ipython/ipython/issues'
88 88 super(ParallelCrashHandler,self).__init__(
89 89 app, contact_name, contact_email, bug_tracker
90 90 )
91 91
92 92
93 93 #-----------------------------------------------------------------------------
94 94 # Main application
95 95 #-----------------------------------------------------------------------------
96 96 base_aliases = {}
97 97 base_aliases.update(base_ip_aliases)
98 98 base_aliases.update({
99 99 'profile_dir' : 'ProfileDir.location',
100 100 'log_level' : 'BaseParallelApplication.log_level',
101 101 'work_dir' : 'BaseParallelApplication.work_dir',
102 102 'log_to_file' : 'BaseParallelApplication.log_to_file',
103 103 'clean_logs' : 'BaseParallelApplication.clean_logs',
104 104 'log_url' : 'BaseParallelApplication.log_url',
105 105 })
106 106
107 107 base_flags = {
108 108 'log-to-file' : (
109 109 {'BaseParallelApplication' : {'log_to_file' : True}},
110 110 "send log output to a file"
111 111 )
112 112 }
113 113 base_flags.update(base_ip_flags)
114 114
115 115 class BaseParallelApplication(BaseIPythonApplication):
116 116 """The base Application for IPython.parallel apps
117 117
118 118 Principle extensions to BaseIPyythonApplication:
119 119
120 120 * work_dir
121 121 * remote logging via pyzmq
122 122 * IOLoop instance
123 123 """
124 124
125 125 crash_handler_class = ParallelCrashHandler
126 126
127 127 def _log_level_default(self):
128 128 # temporarily override default_log_level to INFO
129 129 return logging.INFO
130 130
131 131 work_dir = Unicode(os.getcwdu(), config=True,
132 132 help='Set the working dir for the process.'
133 133 )
134 134 def _work_dir_changed(self, name, old, new):
135 135 self.work_dir = unicode(expand_path(new))
136 136
137 137 log_to_file = Bool(config=True,
138 138 help="whether to log to a file")
139 139
140 140 clean_logs = Bool(False, config=True,
141 141 help="whether to cleanup old logfiles before starting")
142 142
143 143 log_url = Unicode('', config=True,
144 144 help="The ZMQ URL of the iplogger to aggregate logging.")
145 145
146 146 def _config_files_default(self):
147 147 return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
148 148
149 149 loop = Instance('zmq.eventloop.ioloop.IOLoop')
150 150 def _loop_default(self):
151 151 from zmq.eventloop.ioloop import IOLoop
152 152 return IOLoop.instance()
153 153
154 154 aliases = Dict(base_aliases)
155 155 flags = Dict(base_flags)
156 156
157 157 def initialize(self, argv=None):
158 158 """initialize the app"""
159 159 super(BaseParallelApplication, self).initialize(argv)
160 160 self.to_work_dir()
161 161 self.reinit_logging()
162 162
163 163 def to_work_dir(self):
164 164 wd = self.work_dir
165 165 if unicode(wd) != os.getcwdu():
166 166 os.chdir(wd)
167 167 self.log.info("Changing to working dir: %s" % wd)
168 168 # This is the working dir by now.
169 169 sys.path.insert(0, '')
170 170
171 171 def reinit_logging(self):
172 172 # Remove old log files
173 173 log_dir = self.profile_dir.log_dir
174 174 if self.clean_logs:
175 175 for f in os.listdir(log_dir):
176 176 if re.match(r'%s-\d+\.(log|err|out)'%self.name,f):
177 177 os.remove(os.path.join(log_dir, f))
178 178 if self.log_to_file:
179 179 # Start logging to the new log file
180 180 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
181 181 logfile = os.path.join(log_dir, log_filename)
182 182 open_log_file = open(logfile, 'w')
183 183 else:
184 184 open_log_file = None
185 185 if open_log_file is not None:
186 186 self.log.removeHandler(self._log_handler)
187 187 self._log_handler = logging.StreamHandler(open_log_file)
188 188 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
189 189 self._log_handler.setFormatter(self._log_formatter)
190 190 self.log.addHandler(self._log_handler)
191 191
192 192 def write_pid_file(self, overwrite=False):
193 193 """Create a .pid file in the pid_dir with my pid.
194 194
195 195 This must be called after pre_construct, which sets `self.pid_dir`.
196 196 This raises :exc:`PIDFileError` if the pid file exists already.
197 197 """
198 198 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
199 199 if os.path.isfile(pid_file):
200 200 pid = self.get_pid_from_file()
201 201 if not overwrite:
202 202 raise PIDFileError(
203 203 'The pid file [%s] already exists. \nThis could mean that this '
204 204 'server is already running with [pid=%s].' % (pid_file, pid)
205 205 )
206 206 with open(pid_file, 'w') as f:
207 207 self.log.info("Creating pid file: %s" % pid_file)
208 208 f.write(repr(os.getpid())+'\n')
209 209
210 210 def remove_pid_file(self):
211 211 """Remove the pid file.
212 212
213 213 This should be called at shutdown by registering a callback with
214 214 :func:`reactor.addSystemEventTrigger`. This needs to return
215 215 ``None``.
216 216 """
217 217 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
218 218 if os.path.isfile(pid_file):
219 219 try:
220 220 self.log.info("Removing pid file: %s" % pid_file)
221 221 os.remove(pid_file)
222 222 except:
223 223 self.log.warn("Error removing the pid file: %s" % pid_file)
224 224
225 225 def get_pid_from_file(self):
226 226 """Get the pid from the pid file.
227 227
228 228 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
229 229 """
230 230 pid_file = os.path.join(self.profile_dir.pid_dir, self.name + u'.pid')
231 231 if os.path.isfile(pid_file):
232 232 with open(pid_file, 'r') as f:
233 233 pid = int(f.read().strip())
234 234 return pid
235 235 else:
236 236 raise PIDFileError('pid file not found: %s' % pid_file)
237 237
238 238 def check_pid(self, pid):
239 239 if os.name == 'nt':
240 240 try:
241 241 import ctypes
242 242 # returns 0 if no such process (of ours) exists
243 243 # positive int otherwise
244 244 p = ctypes.windll.kernel32.OpenProcess(1,0,pid)
245 245 except Exception:
246 246 self.log.warn(
247 247 "Could not determine whether pid %i is running via `OpenProcess`. "
248 248 " Making the likely assumption that it is."%pid
249 249 )
250 250 return True
251 251 return bool(p)
252 252 else:
253 253 try:
254 254 p = Popen(['ps','x'], stdout=PIPE, stderr=PIPE)
255 255 output,_ = p.communicate()
256 256 except OSError:
257 257 self.log.warn(
258 258 "Could not determine whether pid %i is running via `ps x`. "
259 259 " Making the likely assumption that it is."%pid
260 260 )
261 261 return True
262 262 pids = map(int, re.findall(r'^\W*\d+', output, re.MULTILINE))
263 263 return pid in pids
@@ -1,526 +1,526 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The ipcluster application.
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * MinRK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 import errno
25 25 import logging
26 26 import os
27 27 import re
28 28 import signal
29 29
30 30 from subprocess import check_call, CalledProcessError, PIPE
31 31 import zmq
32 32 from zmq.eventloop import ioloop
33 33
34 34 from IPython.config.application import Application, boolean_flag
35 35 from IPython.config.loader import Config
36 from IPython.core.newapplication import BaseIPythonApplication, ProfileDir
36 from IPython.core.application import BaseIPythonApplication, ProfileDir
37 37 from IPython.utils.daemonize import daemonize
38 38 from IPython.utils.importstring import import_item
39 39 from IPython.utils.traitlets import Int, Unicode, Bool, CFloat, Dict, List
40 40
41 41 from IPython.parallel.apps.baseapp import (
42 42 BaseParallelApplication,
43 43 PIDFileError,
44 44 base_flags, base_aliases
45 45 )
46 46
47 47
48 48 #-----------------------------------------------------------------------------
49 49 # Module level variables
50 50 #-----------------------------------------------------------------------------
51 51
52 52
53 53 default_config_file_name = u'ipcluster_config.py'
54 54
55 55
56 56 _description = """Start an IPython cluster for parallel computing.
57 57
58 58 An IPython cluster consists of 1 controller and 1 or more engines.
59 59 This command automates the startup of these processes using a wide
60 60 range of startup methods (SSH, local processes, PBS, mpiexec,
61 61 Windows HPC Server 2008). To start a cluster with 4 engines on your
62 62 local host simply do 'ipcluster start n=4'. For more complex usage
63 63 you will typically do 'ipcluster create profile=mycluster', then edit
64 64 configuration files, followed by 'ipcluster start profile=mycluster n=4'.
65 65 """
66 66
67 67
68 68 # Exit codes for ipcluster
69 69
70 70 # This will be the exit code if the ipcluster appears to be running because
71 71 # a .pid file exists
72 72 ALREADY_STARTED = 10
73 73
74 74
75 75 # This will be the exit code if ipcluster stop is run, but there is not .pid
76 76 # file to be found.
77 77 ALREADY_STOPPED = 11
78 78
79 79 # This will be the exit code if ipcluster engines is run, but there is not .pid
80 80 # file to be found.
81 81 NO_CLUSTER = 12
82 82
83 83
84 84 #-----------------------------------------------------------------------------
85 85 # Main application
86 86 #-----------------------------------------------------------------------------
87 87 start_help = """Start an IPython cluster for parallel computing
88 88
89 89 Start an ipython cluster by its profile name or cluster
90 90 directory. Cluster directories contain configuration, log and
91 91 security related files and are named using the convention
92 92 'cluster_<profile>' and should be creating using the 'start'
93 93 subcommand of 'ipcluster'. If your cluster directory is in
94 94 the cwd or the ipython directory, you can simply refer to it
95 95 using its profile name, 'ipcluster start n=4 profile=<profile>`,
96 96 otherwise use the 'profile_dir' option.
97 97 """
98 98 stop_help = """Stop a running IPython cluster
99 99
100 100 Stop a running ipython cluster by its profile name or cluster
101 101 directory. Cluster directories are named using the convention
102 102 'cluster_<profile>'. If your cluster directory is in
103 103 the cwd or the ipython directory, you can simply refer to it
104 104 using its profile name, 'ipcluster stop profile=<profile>`, otherwise
105 105 use the 'profile_dir' option.
106 106 """
107 107 engines_help = """Start engines connected to an existing IPython cluster
108 108
109 109 Start one or more engines to connect to an existing Cluster
110 110 by profile name or cluster directory.
111 111 Cluster directories contain configuration, log and
112 112 security related files and are named using the convention
113 113 'cluster_<profile>' and should be creating using the 'start'
114 114 subcommand of 'ipcluster'. If your cluster directory is in
115 115 the cwd or the ipython directory, you can simply refer to it
116 116 using its profile name, 'ipcluster engines n=4 profile=<profile>`,
117 117 otherwise use the 'profile_dir' option.
118 118 """
119 119 create_help = """Create an ipcluster profile by name
120 120
121 121 Create an ipython cluster directory by its profile name or
122 122 cluster directory path. Cluster directories contain
123 123 configuration, log and security related files and are named
124 124 using the convention 'cluster_<profile>'. By default they are
125 125 located in your ipython directory. Once created, you will
126 126 probably need to edit the configuration files in the cluster
127 127 directory to configure your cluster. Most users will create a
128 128 cluster directory by profile name,
129 129 `ipcluster create profile=mycluster`, which will put the directory
130 130 in `<ipython_dir>/cluster_mycluster`.
131 131 """
132 132 list_help = """List available cluster profiles
133 133
134 134 List all available clusters, by cluster directory, that can
135 135 be found in the current working directly or in the ipython
136 136 directory. Cluster directories are named using the convention
137 137 'cluster_<profile>'.
138 138 """
139 139
140 140
141 141 class IPClusterList(BaseIPythonApplication):
142 142 name = u'ipcluster-list'
143 143 description = list_help
144 144
145 145 # empty aliases
146 146 aliases=Dict()
147 147 flags = Dict(base_flags)
148 148
149 149 def _log_level_default(self):
150 150 return 20
151 151
152 152 def list_profile_dirs(self):
153 153 # Find the search paths
154 154 profile_dir_paths = os.environ.get('IPYTHON_PROFILE_PATH','')
155 155 if profile_dir_paths:
156 156 profile_dir_paths = profile_dir_paths.split(':')
157 157 else:
158 158 profile_dir_paths = []
159 159
160 160 ipython_dir = self.ipython_dir
161 161
162 162 paths = [os.getcwd(), ipython_dir] + profile_dir_paths
163 163 paths = list(set(paths))
164 164
165 165 self.log.info('Searching for cluster profiles in paths: %r' % paths)
166 166 for path in paths:
167 167 files = os.listdir(path)
168 168 for f in files:
169 169 full_path = os.path.join(path, f)
170 170 if os.path.isdir(full_path) and f.startswith('profile_') and \
171 171 os.path.isfile(os.path.join(full_path, 'ipcontroller_config.py')):
172 172 profile = f.split('_')[-1]
173 173 start_cmd = 'ipcluster start profile=%s n=4' % profile
174 174 print start_cmd + " ==> " + full_path
175 175
176 176 def start(self):
177 177 self.list_profile_dirs()
178 178
179 179
180 180 # `ipcluster create` will be deprecated when `ipython profile create` or equivalent exists
181 181
182 182 create_flags = {}
183 183 create_flags.update(base_flags)
184 184 create_flags.update(boolean_flag('reset', 'IPClusterCreate.overwrite',
185 185 "reset config files to defaults", "leave existing config files"))
186 186
187 187 class IPClusterCreate(BaseParallelApplication):
188 188 name = u'ipcluster-create'
189 189 description = create_help
190 190 auto_create = Bool(True)
191 191 config_file_name = Unicode(default_config_file_name)
192 192
193 193 flags = Dict(create_flags)
194 194
195 195 aliases = Dict(dict(profile='BaseIPythonApplication.profile'))
196 196
197 197 classes = [ProfileDir]
198 198
199 199
200 200 stop_aliases = dict(
201 201 signal='IPClusterStop.signal',
202 202 profile='BaseIPythonApplication.profile',
203 203 profile_dir='ProfileDir.location',
204 204 )
205 205
206 206 class IPClusterStop(BaseParallelApplication):
207 207 name = u'ipcluster'
208 208 description = stop_help
209 209 config_file_name = Unicode(default_config_file_name)
210 210
211 211 signal = Int(signal.SIGINT, config=True,
212 212 help="signal to use for stopping processes.")
213 213
214 214 aliases = Dict(stop_aliases)
215 215
216 216 def start(self):
217 217 """Start the app for the stop subcommand."""
218 218 try:
219 219 pid = self.get_pid_from_file()
220 220 except PIDFileError:
221 221 self.log.critical(
222 222 'Could not read pid file, cluster is probably not running.'
223 223 )
224 224 # Here I exit with a unusual exit status that other processes
225 225 # can watch for to learn how I existed.
226 226 self.remove_pid_file()
227 227 self.exit(ALREADY_STOPPED)
228 228
229 229 if not self.check_pid(pid):
230 230 self.log.critical(
231 231 'Cluster [pid=%r] is not running.' % pid
232 232 )
233 233 self.remove_pid_file()
234 234 # Here I exit with a unusual exit status that other processes
235 235 # can watch for to learn how I existed.
236 236 self.exit(ALREADY_STOPPED)
237 237
238 238 elif os.name=='posix':
239 239 sig = self.signal
240 240 self.log.info(
241 241 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
242 242 )
243 243 try:
244 244 os.kill(pid, sig)
245 245 except OSError:
246 246 self.log.error("Stopping cluster failed, assuming already dead.",
247 247 exc_info=True)
248 248 self.remove_pid_file()
249 249 elif os.name=='nt':
250 250 try:
251 251 # kill the whole tree
252 252 p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
253 253 except (CalledProcessError, OSError):
254 254 self.log.error("Stopping cluster failed, assuming already dead.",
255 255 exc_info=True)
256 256 self.remove_pid_file()
257 257
258 258 engine_aliases = {}
259 259 engine_aliases.update(base_aliases)
260 260 engine_aliases.update(dict(
261 261 n='IPClusterEngines.n',
262 262 elauncher = 'IPClusterEngines.engine_launcher_class',
263 263 ))
264 264 class IPClusterEngines(BaseParallelApplication):
265 265
266 266 name = u'ipcluster'
267 267 description = engines_help
268 268 usage = None
269 269 config_file_name = Unicode(default_config_file_name)
270 270 default_log_level = logging.INFO
271 271 classes = List()
272 272 def _classes_default(self):
273 273 from IPython.parallel.apps import launcher
274 274 launchers = launcher.all_launchers
275 275 eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
276 276 return [ProfileDir]+eslaunchers
277 277
278 278 n = Int(2, config=True,
279 279 help="The number of engines to start.")
280 280
281 281 engine_launcher_class = Unicode('LocalEngineSetLauncher',
282 282 config=True,
283 283 help="The class for launching a set of Engines."
284 284 )
285 285 daemonize = Bool(False, config=True,
286 286 help='Daemonize the ipcluster program. This implies --log-to-file')
287 287
288 288 def _daemonize_changed(self, name, old, new):
289 289 if new:
290 290 self.log_to_file = True
291 291
292 292 aliases = Dict(engine_aliases)
293 293 # flags = Dict(flags)
294 294 _stopping = False
295 295
296 296 def initialize(self, argv=None):
297 297 super(IPClusterEngines, self).initialize(argv)
298 298 self.init_signal()
299 299 self.init_launchers()
300 300
301 301 def init_launchers(self):
302 302 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
303 303 self.engine_launcher.on_stop(lambda r: self.loop.stop())
304 304
305 305 def init_signal(self):
306 306 # Setup signals
307 307 signal.signal(signal.SIGINT, self.sigint_handler)
308 308
309 309 def build_launcher(self, clsname):
310 310 """import and instantiate a Launcher based on importstring"""
311 311 if '.' not in clsname:
312 312 # not a module, presume it's the raw name in apps.launcher
313 313 clsname = 'IPython.parallel.apps.launcher.'+clsname
314 314 # print repr(clsname)
315 315 klass = import_item(clsname)
316 316
317 317 launcher = klass(
318 318 work_dir=self.profile_dir.location, config=self.config, log=self.log
319 319 )
320 320 return launcher
321 321
322 322 def start_engines(self):
323 323 self.log.info("Starting %i engines"%self.n)
324 324 self.engine_launcher.start(
325 325 self.n,
326 326 self.profile_dir.location
327 327 )
328 328
329 329 def stop_engines(self):
330 330 self.log.info("Stopping Engines...")
331 331 if self.engine_launcher.running:
332 332 d = self.engine_launcher.stop()
333 333 return d
334 334 else:
335 335 return None
336 336
337 337 def stop_launchers(self, r=None):
338 338 if not self._stopping:
339 339 self._stopping = True
340 340 self.log.error("IPython cluster: stopping")
341 341 self.stop_engines()
342 342 # Wait a few seconds to let things shut down.
343 343 dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
344 344 dc.start()
345 345
346 346 def sigint_handler(self, signum, frame):
347 347 self.log.debug("SIGINT received, stopping launchers...")
348 348 self.stop_launchers()
349 349
350 350 def start_logging(self):
351 351 # Remove old log files of the controller and engine
352 352 if self.clean_logs:
353 353 log_dir = self.profile_dir.log_dir
354 354 for f in os.listdir(log_dir):
355 355 if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
356 356 os.remove(os.path.join(log_dir, f))
357 357 # This will remove old log files for ipcluster itself
358 358 # super(IPBaseParallelApplication, self).start_logging()
359 359
360 360 def start(self):
361 361 """Start the app for the engines subcommand."""
362 362 self.log.info("IPython cluster: started")
363 363 # First see if the cluster is already running
364 364
365 365 # Now log and daemonize
366 366 self.log.info(
367 367 'Starting engines with [daemon=%r]' % self.daemonize
368 368 )
369 369 # TODO: Get daemonize working on Windows or as a Windows Server.
370 370 if self.daemonize:
371 371 if os.name=='posix':
372 372 daemonize()
373 373
374 374 dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
375 375 dc.start()
376 376 # Now write the new pid file AFTER our new forked pid is active.
377 377 # self.write_pid_file()
378 378 try:
379 379 self.loop.start()
380 380 except KeyboardInterrupt:
381 381 pass
382 382 except zmq.ZMQError as e:
383 383 if e.errno == errno.EINTR:
384 384 pass
385 385 else:
386 386 raise
387 387
388 388 start_aliases = {}
389 389 start_aliases.update(engine_aliases)
390 390 start_aliases.update(dict(
391 391 delay='IPClusterStart.delay',
392 392 clean_logs='IPClusterStart.clean_logs',
393 393 ))
394 394
395 395 class IPClusterStart(IPClusterEngines):
396 396
397 397 name = u'ipcluster'
398 398 description = start_help
399 399 default_log_level = logging.INFO
400 400 auto_create = Bool(True, config=True,
401 401 help="whether to create the profile_dir if it doesn't exist")
402 402 classes = List()
403 403 def _classes_default(self,):
404 404 from IPython.parallel.apps import launcher
405 405 return [ProfileDir]+launcher.all_launchers
406 406
407 407 clean_logs = Bool(True, config=True,
408 408 help="whether to cleanup old logs before starting")
409 409
410 410 delay = CFloat(1., config=True,
411 411 help="delay (in s) between starting the controller and the engines")
412 412
413 413 controller_launcher_class = Unicode('LocalControllerLauncher',
414 414 config=True,
415 415 help="The class for launching a Controller."
416 416 )
417 417 reset = Bool(False, config=True,
418 418 help="Whether to reset config files as part of '--create'."
419 419 )
420 420
421 421 # flags = Dict(flags)
422 422 aliases = Dict(start_aliases)
423 423
424 424 def init_launchers(self):
425 425 self.controller_launcher = self.build_launcher(self.controller_launcher_class)
426 426 self.engine_launcher = self.build_launcher(self.engine_launcher_class)
427 427 self.controller_launcher.on_stop(self.stop_launchers)
428 428
429 429 def start_controller(self):
430 430 self.controller_launcher.start(
431 431 self.profile_dir.location
432 432 )
433 433
434 434 def stop_controller(self):
435 435 # self.log.info("In stop_controller")
436 436 if self.controller_launcher and self.controller_launcher.running:
437 437 return self.controller_launcher.stop()
438 438
439 439 def stop_launchers(self, r=None):
440 440 if not self._stopping:
441 441 self.stop_controller()
442 442 super(IPClusterStart, self).stop_launchers()
443 443
444 444 def start(self):
445 445 """Start the app for the start subcommand."""
446 446 # First see if the cluster is already running
447 447 try:
448 448 pid = self.get_pid_from_file()
449 449 except PIDFileError:
450 450 pass
451 451 else:
452 452 if self.check_pid(pid):
453 453 self.log.critical(
454 454 'Cluster is already running with [pid=%s]. '
455 455 'use "ipcluster stop" to stop the cluster.' % pid
456 456 )
457 457 # Here I exit with a unusual exit status that other processes
458 458 # can watch for to learn how I existed.
459 459 self.exit(ALREADY_STARTED)
460 460 else:
461 461 self.remove_pid_file()
462 462
463 463
464 464 # Now log and daemonize
465 465 self.log.info(
466 466 'Starting ipcluster with [daemon=%r]' % self.daemonize
467 467 )
468 468 # TODO: Get daemonize working on Windows or as a Windows Server.
469 469 if self.daemonize:
470 470 if os.name=='posix':
471 471 daemonize()
472 472
473 473 dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
474 474 dc.start()
475 475 dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
476 476 dc.start()
477 477 # Now write the new pid file AFTER our new forked pid is active.
478 478 self.write_pid_file()
479 479 try:
480 480 self.loop.start()
481 481 except KeyboardInterrupt:
482 482 pass
483 483 except zmq.ZMQError as e:
484 484 if e.errno == errno.EINTR:
485 485 pass
486 486 else:
487 487 raise
488 488 finally:
489 489 self.remove_pid_file()
490 490
491 491 base='IPython.parallel.apps.ipclusterapp.IPCluster'
492 492
493 493 class IPBaseParallelApplication(Application):
494 494 name = u'ipcluster'
495 495 description = _description
496 496
497 497 subcommands = {'create' : (base+'Create', create_help),
498 498 'list' : (base+'List', list_help),
499 499 'start' : (base+'Start', start_help),
500 500 'stop' : (base+'Stop', stop_help),
501 501 'engines' : (base+'Engines', engines_help),
502 502 }
503 503
504 504 # no aliases or flags for parent App
505 505 aliases = Dict()
506 506 flags = Dict()
507 507
508 508 def start(self):
509 509 if self.subapp is None:
510 510 print "No subcommand specified! Must specify one of: %s"%(self.subcommands.keys())
511 511 print
512 512 self.print_subcommands()
513 513 self.exit(1)
514 514 else:
515 515 return self.subapp.start()
516 516
517 517 def launch_new_instance():
518 518 """Create and run the IPython cluster."""
519 519 app = IPBaseParallelApplication.instance()
520 520 app.initialize()
521 521 app.start()
522 522
523 523
524 524 if __name__ == '__main__':
525 525 launch_new_instance()
526 526
@@ -1,408 +1,408 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The IPython controller application.
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * MinRK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 from __future__ import with_statement
25 25
26 26 import os
27 27 import socket
28 28 import stat
29 29 import sys
30 30 import uuid
31 31
32 32 from multiprocessing import Process
33 33
34 34 import zmq
35 35 from zmq.devices import ProcessMonitoredQueue
36 36 from zmq.log.handlers import PUBHandler
37 37 from zmq.utils import jsonapi as json
38 38
39 39 from IPython.config.application import boolean_flag
40 from IPython.core.newapplication import ProfileDir
40 from IPython.core.application import ProfileDir
41 41
42 42 from IPython.parallel.apps.baseapp import (
43 43 BaseParallelApplication,
44 44 base_flags
45 45 )
46 46 from IPython.utils.importstring import import_item
47 47 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict
48 48
49 49 # from IPython.parallel.controller.controller import ControllerFactory
50 50 from IPython.zmq.session import Session
51 51 from IPython.parallel.controller.heartmonitor import HeartMonitor
52 52 from IPython.parallel.controller.hub import HubFactory
53 53 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
54 54 from IPython.parallel.controller.sqlitedb import SQLiteDB
55 55
56 56 from IPython.parallel.util import signal_children, split_url
57 57
58 58 # conditional import of MongoDB backend class
59 59
60 60 try:
61 61 from IPython.parallel.controller.mongodb import MongoDB
62 62 except ImportError:
63 63 maybe_mongo = []
64 64 else:
65 65 maybe_mongo = [MongoDB]
66 66
67 67
68 68 #-----------------------------------------------------------------------------
69 69 # Module level variables
70 70 #-----------------------------------------------------------------------------
71 71
72 72
73 73 #: The default config file name for this application
74 74 default_config_file_name = u'ipcontroller_config.py'
75 75
76 76
77 77 _description = """Start the IPython controller for parallel computing.
78 78
79 79 The IPython controller provides a gateway between the IPython engines and
80 80 clients. The controller needs to be started before the engines and can be
81 81 configured using command line options or using a cluster directory. Cluster
82 82 directories contain config, log and security files and are usually located in
83 83 your ipython directory and named as "cluster_<profile>". See the `profile`
84 84 and `profile_dir` options for details.
85 85 """
86 86
87 87
88 88
89 89
90 90 #-----------------------------------------------------------------------------
91 91 # The main application
92 92 #-----------------------------------------------------------------------------
93 93 flags = {}
94 94 flags.update(base_flags)
95 95 flags.update({
96 96 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
97 97 'Use threads instead of processes for the schedulers'),
98 98 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
99 99 'use the SQLiteDB backend'),
100 100 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
101 101 'use the MongoDB backend'),
102 102 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
103 103 'use the in-memory DictDB backend'),
104 104 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
105 105 'reuse existing json connection files')
106 106 })
107 107
108 108 flags.update(boolean_flag('secure', 'IPControllerApp.secure',
109 109 "Use HMAC digests for authentication of messages.",
110 110 "Don't authenticate messages."
111 111 ))
112 112
113 113 class IPControllerApp(BaseParallelApplication):
114 114
115 115 name = u'ipcontroller'
116 116 description = _description
117 117 config_file_name = Unicode(default_config_file_name)
118 118 classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
119 119
120 120 # change default to True
121 121 auto_create = Bool(True, config=True,
122 122 help="""Whether to create profile dir if it doesn't exist.""")
123 123
124 124 reuse_files = Bool(False, config=True,
125 125 help='Whether to reuse existing json connection files.'
126 126 )
127 127 secure = Bool(True, config=True,
128 128 help='Whether to use HMAC digests for extra message authentication.'
129 129 )
130 130 ssh_server = Unicode(u'', config=True,
131 131 help="""ssh url for clients to use when connecting to the Controller
132 132 processes. It should be of the form: [user@]server[:port]. The
133 133 Controller's listening addresses must be accessible from the ssh server""",
134 134 )
135 135 location = Unicode(u'', config=True,
136 136 help="""The external IP or domain name of the Controller, used for disambiguating
137 137 engine and client connections.""",
138 138 )
139 139 import_statements = List([], config=True,
140 140 help="import statements to be run at startup. Necessary in some environments"
141 141 )
142 142
143 143 use_threads = Bool(False, config=True,
144 144 help='Use threads instead of processes for the schedulers',
145 145 )
146 146
147 147 # internal
148 148 children = List()
149 149 mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
150 150
151 151 def _use_threads_changed(self, name, old, new):
152 152 self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
153 153
154 154 aliases = Dict(dict(
155 155 log_level = 'IPControllerApp.log_level',
156 156 log_url = 'IPControllerApp.log_url',
157 157 reuse_files = 'IPControllerApp.reuse_files',
158 158 secure = 'IPControllerApp.secure',
159 159 ssh = 'IPControllerApp.ssh_server',
160 160 use_threads = 'IPControllerApp.use_threads',
161 161 import_statements = 'IPControllerApp.import_statements',
162 162 location = 'IPControllerApp.location',
163 163
164 164 ident = 'Session.session',
165 165 user = 'Session.username',
166 166 exec_key = 'Session.keyfile',
167 167
168 168 url = 'HubFactory.url',
169 169 ip = 'HubFactory.ip',
170 170 transport = 'HubFactory.transport',
171 171 port = 'HubFactory.regport',
172 172
173 173 ping = 'HeartMonitor.period',
174 174
175 175 scheme = 'TaskScheduler.scheme_name',
176 176 hwm = 'TaskScheduler.hwm',
177 177
178 178
179 179 profile = "BaseIPythonApplication.profile",
180 180 profile_dir = 'ProfileDir.location',
181 181
182 182 ))
183 183 flags = Dict(flags)
184 184
185 185
186 186 def save_connection_dict(self, fname, cdict):
187 187 """save a connection dict to json file."""
188 188 c = self.config
189 189 url = cdict['url']
190 190 location = cdict['location']
191 191 if not location:
192 192 try:
193 193 proto,ip,port = split_url(url)
194 194 except AssertionError:
195 195 pass
196 196 else:
197 197 location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
198 198 cdict['location'] = location
199 199 fname = os.path.join(self.profile_dir.security_dir, fname)
200 200 with open(fname, 'w') as f:
201 201 f.write(json.dumps(cdict, indent=2))
202 202 os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
203 203
204 204 def load_config_from_json(self):
205 205 """load config from existing json connector files."""
206 206 c = self.config
207 207 # load from engine config
208 208 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
209 209 cfg = json.loads(f.read())
210 210 key = c.Session.key = cfg['exec_key']
211 211 xport,addr = cfg['url'].split('://')
212 212 c.HubFactory.engine_transport = xport
213 213 ip,ports = addr.split(':')
214 214 c.HubFactory.engine_ip = ip
215 215 c.HubFactory.regport = int(ports)
216 216 self.location = cfg['location']
217 217
218 218 # load client config
219 219 with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
220 220 cfg = json.loads(f.read())
221 221 assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
222 222 xport,addr = cfg['url'].split('://')
223 223 c.HubFactory.client_transport = xport
224 224 ip,ports = addr.split(':')
225 225 c.HubFactory.client_ip = ip
226 226 self.ssh_server = cfg['ssh']
227 227 assert int(ports) == c.HubFactory.regport, "regport mismatch"
228 228
229 229 def init_hub(self):
230 230 c = self.config
231 231
232 232 self.do_import_statements()
233 233 reusing = self.reuse_files
234 234 if reusing:
235 235 try:
236 236 self.load_config_from_json()
237 237 except (AssertionError,IOError):
238 238 reusing=False
239 239 # check again, because reusing may have failed:
240 240 if reusing:
241 241 pass
242 242 elif self.secure:
243 243 key = str(uuid.uuid4())
244 244 # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
245 245 # with open(keyfile, 'w') as f:
246 246 # f.write(key)
247 247 # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
248 248 c.Session.key = key
249 249 else:
250 250 key = c.Session.key = ''
251 251
252 252 try:
253 253 self.factory = HubFactory(config=c, log=self.log)
254 254 # self.start_logging()
255 255 self.factory.init_hub()
256 256 except:
257 257 self.log.error("Couldn't construct the Controller", exc_info=True)
258 258 self.exit(1)
259 259
260 260 if not reusing:
261 261 # save to new json config files
262 262 f = self.factory
263 263 cdict = {'exec_key' : key,
264 264 'ssh' : self.ssh_server,
265 265 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
266 266 'location' : self.location
267 267 }
268 268 self.save_connection_dict('ipcontroller-client.json', cdict)
269 269 edict = cdict
270 270 edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
271 271 self.save_connection_dict('ipcontroller-engine.json', edict)
272 272
273 273 #
274 274 def init_schedulers(self):
275 275 children = self.children
276 276 mq = import_item(str(self.mq_class))
277 277
278 278 hub = self.factory
279 279 # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
280 280 # IOPub relay (in a Process)
281 281 q = mq(zmq.PUB, zmq.SUB, zmq.PUB, 'N/A','iopub')
282 282 q.bind_in(hub.client_info['iopub'])
283 283 q.bind_out(hub.engine_info['iopub'])
284 284 q.setsockopt_out(zmq.SUBSCRIBE, '')
285 285 q.connect_mon(hub.monitor_url)
286 286 q.daemon=True
287 287 children.append(q)
288 288
289 289 # Multiplexer Queue (in a Process)
290 290 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'in', 'out')
291 291 q.bind_in(hub.client_info['mux'])
292 292 q.setsockopt_in(zmq.IDENTITY, 'mux')
293 293 q.bind_out(hub.engine_info['mux'])
294 294 q.connect_mon(hub.monitor_url)
295 295 q.daemon=True
296 296 children.append(q)
297 297
298 298 # Control Queue (in a Process)
299 299 q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'incontrol', 'outcontrol')
300 300 q.bind_in(hub.client_info['control'])
301 301 q.setsockopt_in(zmq.IDENTITY, 'control')
302 302 q.bind_out(hub.engine_info['control'])
303 303 q.connect_mon(hub.monitor_url)
304 304 q.daemon=True
305 305 children.append(q)
306 306 try:
307 307 scheme = self.config.TaskScheduler.scheme_name
308 308 except AttributeError:
309 309 scheme = TaskScheduler.scheme_name.get_default_value()
310 310 # Task Queue (in a Process)
311 311 if scheme == 'pure':
312 312 self.log.warn("task::using pure XREQ Task scheduler")
313 313 q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, 'intask', 'outtask')
314 314 # q.setsockopt_out(zmq.HWM, hub.hwm)
315 315 q.bind_in(hub.client_info['task'][1])
316 316 q.setsockopt_in(zmq.IDENTITY, 'task')
317 317 q.bind_out(hub.engine_info['task'])
318 318 q.connect_mon(hub.monitor_url)
319 319 q.daemon=True
320 320 children.append(q)
321 321 elif scheme == 'none':
322 322 self.log.warn("task::using no Task scheduler")
323 323
324 324 else:
325 325 self.log.info("task::using Python %s Task scheduler"%scheme)
326 326 sargs = (hub.client_info['task'][1], hub.engine_info['task'],
327 327 hub.monitor_url, hub.client_info['notification'])
328 328 kwargs = dict(logname='scheduler', loglevel=self.log_level,
329 329 log_url = self.log_url, config=dict(self.config))
330 330 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
331 331 q.daemon=True
332 332 children.append(q)
333 333
334 334
335 335 def save_urls(self):
336 336 """save the registration urls to files."""
337 337 c = self.config
338 338
339 339 sec_dir = self.profile_dir.security_dir
340 340 cf = self.factory
341 341
342 342 with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
343 343 f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))
344 344
345 345 with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
346 346 f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))
347 347
348 348
349 349 def do_import_statements(self):
350 350 statements = self.import_statements
351 351 for s in statements:
352 352 try:
353 353 self.log.msg("Executing statement: '%s'" % s)
354 354 exec s in globals(), locals()
355 355 except:
356 356 self.log.msg("Error running statement: %s" % s)
357 357
358 358 def forward_logging(self):
359 359 if self.log_url:
360 360 self.log.info("Forwarding logging to %s"%self.log_url)
361 361 context = zmq.Context.instance()
362 362 lsock = context.socket(zmq.PUB)
363 363 lsock.connect(self.log_url)
364 364 handler = PUBHandler(lsock)
365 365 self.log.removeHandler(self._log_handler)
366 366 handler.root_topic = 'controller'
367 367 handler.setLevel(self.log_level)
368 368 self.log.addHandler(handler)
369 369 self._log_handler = handler
370 370 # #
371 371
372 372 def initialize(self, argv=None):
373 373 super(IPControllerApp, self).initialize(argv)
374 374 self.forward_logging()
375 375 self.init_hub()
376 376 self.init_schedulers()
377 377
378 378 def start(self):
379 379 # Start the subprocesses:
380 380 self.factory.start()
381 381 child_procs = []
382 382 for child in self.children:
383 383 child.start()
384 384 if isinstance(child, ProcessMonitoredQueue):
385 385 child_procs.append(child.launcher)
386 386 elif isinstance(child, Process):
387 387 child_procs.append(child)
388 388 if child_procs:
389 389 signal_children(child_procs)
390 390
391 391 self.write_pid_file(overwrite=True)
392 392
393 393 try:
394 394 self.factory.loop.start()
395 395 except KeyboardInterrupt:
396 396 self.log.critical("Interrupted, Exiting...\n")
397 397
398 398
399 399
400 400 def launch_new_instance():
401 401 """Create and run the IPython controller"""
402 402 app = IPControllerApp.instance()
403 403 app.initialize()
404 404 app.start()
405 405
406 406
407 407 if __name__ == '__main__':
408 408 launch_new_instance()
@@ -1,276 +1,276 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The IPython engine application
5 5
6 6 Authors:
7 7
8 8 * Brian Granger
9 9 * MinRK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008-2011 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 import json
25 25 import os
26 26 import sys
27 27
28 28 import zmq
29 29 from zmq.eventloop import ioloop
30 30
31 from IPython.core.newapplication import ProfileDir
31 from IPython.core.application import ProfileDir
32 32 from IPython.parallel.apps.baseapp import BaseParallelApplication
33 33 from IPython.zmq.log import EnginePUBHandler
34 34
35 35 from IPython.config.configurable import Configurable
36 36 from IPython.zmq.session import Session
37 37 from IPython.parallel.engine.engine import EngineFactory
38 38 from IPython.parallel.engine.streamkernel import Kernel
39 39 from IPython.parallel.util import disambiguate_url
40 40
41 41 from IPython.utils.importstring import import_item
42 42 from IPython.utils.traitlets import Bool, Unicode, Dict, List
43 43
44 44
45 45 #-----------------------------------------------------------------------------
46 46 # Module level variables
47 47 #-----------------------------------------------------------------------------
48 48
49 49 #: The default config file name for this application
50 50 default_config_file_name = u'ipengine_config.py'
51 51
52 52 _description = """Start an IPython engine for parallel computing.
53 53
54 54 IPython engines run in parallel and perform computations on behalf of a client
55 55 and controller. A controller needs to be started before the engines. The
56 56 engine can be configured using command line options or using a cluster
57 57 directory. Cluster directories contain config, log and security files and are
58 58 usually located in your ipython directory and named as "cluster_<profile>".
59 59 See the `profile` and `profile_dir` options for details.
60 60 """
61 61
62 62
63 63 #-----------------------------------------------------------------------------
64 64 # MPI configuration
65 65 #-----------------------------------------------------------------------------
66 66
67 67 mpi4py_init = """from mpi4py import MPI as mpi
68 68 mpi.size = mpi.COMM_WORLD.Get_size()
69 69 mpi.rank = mpi.COMM_WORLD.Get_rank()
70 70 """
71 71
72 72
73 73 pytrilinos_init = """from PyTrilinos import Epetra
74 74 class SimpleStruct:
75 75 pass
76 76 mpi = SimpleStruct()
77 77 mpi.rank = 0
78 78 mpi.size = 0
79 79 """
80 80
81 81 class MPI(Configurable):
82 82 """Configurable for MPI initialization"""
83 83 use = Unicode('', config=True,
84 84 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
85 85 )
86 86
87 87 def _on_use_changed(self, old, new):
88 88 # load default init script if it's not set
89 89 if not self.init_script:
90 90 self.init_script = self.default_inits.get(new, '')
91 91
92 92 init_script = Unicode('', config=True,
93 93 help="Initialization code for MPI")
94 94
95 95 default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init},
96 96 config=True)
97 97
98 98
99 99 #-----------------------------------------------------------------------------
100 100 # Main application
101 101 #-----------------------------------------------------------------------------
102 102
103 103
104 104 class IPEngineApp(BaseParallelApplication):
105 105
106 106 app_name = Unicode(u'ipengine')
107 107 description = Unicode(_description)
108 108 config_file_name = Unicode(default_config_file_name)
109 109 classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
110 110
111 111 startup_script = Unicode(u'', config=True,
112 112 help='specify a script to be run at startup')
113 113 startup_command = Unicode('', config=True,
114 114 help='specify a command to be run at startup')
115 115
116 116 url_file = Unicode(u'', config=True,
117 117 help="""The full location of the file containing the connection information for
118 118 the controller. If this is not given, the file must be in the
119 119 security directory of the cluster directory. This location is
120 120 resolved using the `profile` or `profile_dir` options.""",
121 121 )
122 122
123 123 url_file_name = Unicode(u'ipcontroller-engine.json')
124 124 log_url = Unicode('', config=True,
125 125 help="""The URL for the iploggerapp instance, for forwarding
126 126 logging to a central location.""")
127 127
128 128 aliases = Dict(dict(
129 129 file = 'IPEngineApp.url_file',
130 130 c = 'IPEngineApp.startup_command',
131 131 s = 'IPEngineApp.startup_script',
132 132
133 133 ident = 'Session.session',
134 134 user = 'Session.username',
135 135 exec_key = 'Session.keyfile',
136 136
137 137 url = 'EngineFactory.url',
138 138 ip = 'EngineFactory.ip',
139 139 transport = 'EngineFactory.transport',
140 140 port = 'EngineFactory.regport',
141 141 location = 'EngineFactory.location',
142 142
143 143 timeout = 'EngineFactory.timeout',
144 144
145 145 profile = "IPEngineApp.profile",
146 146 profile_dir = 'ProfileDir.location',
147 147
148 148 mpi = 'MPI.use',
149 149
150 150 log_level = 'IPEngineApp.log_level',
151 151 log_url = 'IPEngineApp.log_url'
152 152 ))
153 153
154 154 # def find_key_file(self):
155 155 # """Set the key file.
156 156 #
157 157 # Here we don't try to actually see if it exists for is valid as that
158 158 # is hadled by the connection logic.
159 159 # """
160 160 # config = self.master_config
161 161 # # Find the actual controller key file
162 162 # if not config.Global.key_file:
163 163 # try_this = os.path.join(
164 164 # config.Global.profile_dir,
165 165 # config.Global.security_dir,
166 166 # config.Global.key_file_name
167 167 # )
168 168 # config.Global.key_file = try_this
169 169
170 170 def find_url_file(self):
171 171 """Set the key file.
172 172
173 173 Here we don't try to actually see if it exists for is valid as that
174 174 is hadled by the connection logic.
175 175 """
176 176 config = self.config
177 177 # Find the actual controller key file
178 178 if not self.url_file:
179 179 self.url_file = os.path.join(
180 180 self.profile_dir.security_dir,
181 181 self.url_file_name
182 182 )
183 183 def init_engine(self):
184 184 # This is the working dir by now.
185 185 sys.path.insert(0, '')
186 186 config = self.config
187 187 # print config
188 188 self.find_url_file()
189 189
190 190 # if os.path.exists(config.Global.key_file) and config.Global.secure:
191 191 # config.SessionFactory.exec_key = config.Global.key_file
192 192 if os.path.exists(self.url_file):
193 193 with open(self.url_file) as f:
194 194 d = json.loads(f.read())
195 195 for k,v in d.iteritems():
196 196 if isinstance(v, unicode):
197 197 d[k] = v.encode()
198 198 if d['exec_key']:
199 199 config.Session.key = d['exec_key']
200 200 d['url'] = disambiguate_url(d['url'], d['location'])
201 201 config.EngineFactory.url = d['url']
202 202 config.EngineFactory.location = d['location']
203 203
204 204 try:
205 205 exec_lines = config.Kernel.exec_lines
206 206 except AttributeError:
207 207 config.Kernel.exec_lines = []
208 208 exec_lines = config.Kernel.exec_lines
209 209
210 210 if self.startup_script:
211 211 enc = sys.getfilesystemencoding() or 'utf8'
212 212 cmd="execfile(%r)"%self.startup_script.encode(enc)
213 213 exec_lines.append(cmd)
214 214 if self.startup_command:
215 215 exec_lines.append(self.startup_command)
216 216
217 217 # Create the underlying shell class and Engine
218 218 # shell_class = import_item(self.master_config.Global.shell_class)
219 219 # print self.config
220 220 try:
221 221 self.engine = EngineFactory(config=config, log=self.log)
222 222 except:
223 223 self.log.error("Couldn't start the Engine", exc_info=True)
224 224 self.exit(1)
225 225
226 226 def forward_logging(self):
227 227 if self.log_url:
228 228 self.log.info("Forwarding logging to %s"%self.log_url)
229 229 context = self.engine.context
230 230 lsock = context.socket(zmq.PUB)
231 231 lsock.connect(self.log_url)
232 232 self.log.removeHandler(self._log_handler)
233 233 handler = EnginePUBHandler(self.engine, lsock)
234 234 handler.setLevel(self.log_level)
235 235 self.log.addHandler(handler)
236 236 self._log_handler = handler
237 237 #
238 238 def init_mpi(self):
239 239 global mpi
240 240 self.mpi = MPI(config=self.config)
241 241
242 242 mpi_import_statement = self.mpi.init_script
243 243 if mpi_import_statement:
244 244 try:
245 245 self.log.info("Initializing MPI:")
246 246 self.log.info(mpi_import_statement)
247 247 exec mpi_import_statement in globals()
248 248 except:
249 249 mpi = None
250 250 else:
251 251 mpi = None
252 252
253 253 def initialize(self, argv=None):
254 254 super(IPEngineApp, self).initialize(argv)
255 255 self.init_mpi()
256 256 self.init_engine()
257 257 self.forward_logging()
258 258
259 259 def start(self):
260 260 self.engine.start()
261 261 try:
262 262 self.engine.loop.start()
263 263 except KeyboardInterrupt:
264 264 self.log.critical("Engine Interrupted, shutting down...\n")
265 265
266 266
267 267 def launch_new_instance():
268 268 """Create and run the IPython engine"""
269 269 app = IPEngineApp.instance()
270 270 app.initialize()
271 271 app.start()
272 272
273 273
274 274 if __name__ == '__main__':
275 275 launch_new_instance()
276 276
@@ -1,101 +1,101 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 A simple IPython logger application
5 5
6 6 Authors:
7 7
8 8 * MinRK
9 9
10 10 """
11 11
12 12 #-----------------------------------------------------------------------------
13 13 # Copyright (C) 2011 The IPython Development Team
14 14 #
15 15 # Distributed under the terms of the BSD License. The full license is in
16 16 # the file COPYING, distributed as part of this software.
17 17 #-----------------------------------------------------------------------------
18 18
19 19 #-----------------------------------------------------------------------------
20 20 # Imports
21 21 #-----------------------------------------------------------------------------
22 22
23 23 import os
24 24 import sys
25 25
26 26 import zmq
27 27
28 from IPython.core.newapplication import ProfileDir
28 from IPython.core.application import ProfileDir
29 29 from IPython.utils.traitlets import Bool, Dict, Unicode
30 30
31 31 from IPython.parallel.apps.baseapp import (
32 32 BaseParallelApplication,
33 33 base_aliases
34 34 )
35 35 from IPython.parallel.apps.logwatcher import LogWatcher
36 36
37 37 #-----------------------------------------------------------------------------
38 38 # Module level variables
39 39 #-----------------------------------------------------------------------------
40 40
41 41 #: The default config file name for this application
42 42 default_config_file_name = u'iplogger_config.py'
43 43
44 44 _description = """Start an IPython logger for parallel computing.
45 45
46 46 IPython controllers and engines (and your own processes) can broadcast log messages
47 47 by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
48 48 logger can be configured using command line options or using a cluster
49 49 directory. Cluster directories contain config, log and security files and are
50 50 usually located in your ipython directory and named as "cluster_<profile>".
51 51 See the `profile` and `profile_dir` options for details.
52 52 """
53 53
54 54
55 55 #-----------------------------------------------------------------------------
56 56 # Main application
57 57 #-----------------------------------------------------------------------------
58 58 aliases = {}
59 59 aliases.update(base_aliases)
60 60 aliases.update(dict(url='LogWatcher.url', topics='LogWatcher.topics'))
61 61
62 62 class IPLoggerApp(BaseParallelApplication):
63 63
64 64 name = u'iploggerz'
65 65 description = _description
66 66 config_file_name = Unicode(default_config_file_name)
67 67
68 68 classes = [LogWatcher, ProfileDir]
69 69 aliases = Dict(aliases)
70 70
71 71 def initialize(self, argv=None):
72 72 super(IPLoggerApp, self).initialize(argv)
73 73 self.init_watcher()
74 74
75 75 def init_watcher(self):
76 76 try:
77 77 self.watcher = LogWatcher(config=self.config, log=self.log)
78 78 except:
79 79 self.log.error("Couldn't start the LogWatcher", exc_info=True)
80 80 self.exit(1)
81 81 self.log.info("Listening for log messages on %r"%self.watcher.url)
82 82
83 83
84 84 def start(self):
85 85 self.watcher.start()
86 86 try:
87 87 self.watcher.loop.start()
88 88 except KeyboardInterrupt:
89 89 self.log.critical("Logging Interrupted, shutting down...\n")
90 90
91 91
92 92 def launch_new_instance():
93 93 """Create and run the IPython LogWatcher"""
94 94 app = IPLoggerApp.instance()
95 95 app.initialize()
96 96 app.start()
97 97
98 98
99 99 if __name__ == '__main__':
100 100 launch_new_instance()
101 101
@@ -1,1373 +1,1373 b''
1 1 """A semi-synchronous Client for the ZMQ cluster
2 2
3 3 Authors:
4 4
5 5 * MinRK
6 6 """
7 7 #-----------------------------------------------------------------------------
8 8 # Copyright (C) 2010-2011 The IPython Development Team
9 9 #
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 import os
19 19 import json
20 20 import time
21 21 import warnings
22 22 from datetime import datetime
23 23 from getpass import getpass
24 24 from pprint import pprint
25 25
26 26 pjoin = os.path.join
27 27
28 28 import zmq
29 29 # from zmq.eventloop import ioloop, zmqstream
30 30
31 31 from IPython.utils.path import get_ipython_dir
32 32 from IPython.utils.traitlets import (HasTraits, Int, Instance, Unicode,
33 33 Dict, List, Bool, Set)
34 34 from IPython.external.decorator import decorator
35 35 from IPython.external.ssh import tunnel
36 36
37 37 from IPython.parallel import error
38 38 from IPython.parallel import util
39 39
40 40 from IPython.zmq.session import Session, Message
41 41
42 42 from .asyncresult import AsyncResult, AsyncHubResult
43 from IPython.core.newapplication import ProfileDir, ProfileDirError
43 from IPython.core.application import ProfileDir, ProfileDirError
44 44 from .view import DirectView, LoadBalancedView
45 45
46 46 #--------------------------------------------------------------------------
47 47 # Decorators for Client methods
48 48 #--------------------------------------------------------------------------
49 49
50 50 @decorator
51 51 def spin_first(f, self, *args, **kwargs):
52 52 """Call spin() to sync state prior to calling the method."""
53 53 self.spin()
54 54 return f(self, *args, **kwargs)
55 55
56 56
57 57 #--------------------------------------------------------------------------
58 58 # Classes
59 59 #--------------------------------------------------------------------------
60 60
61 61 class Metadata(dict):
62 62 """Subclass of dict for initializing metadata values.
63 63
64 64 Attribute access works on keys.
65 65
66 66 These objects have a strict set of keys - errors will raise if you try
67 67 to add new keys.
68 68 """
69 69 def __init__(self, *args, **kwargs):
70 70 dict.__init__(self)
71 71 md = {'msg_id' : None,
72 72 'submitted' : None,
73 73 'started' : None,
74 74 'completed' : None,
75 75 'received' : None,
76 76 'engine_uuid' : None,
77 77 'engine_id' : None,
78 78 'follow' : None,
79 79 'after' : None,
80 80 'status' : None,
81 81
82 82 'pyin' : None,
83 83 'pyout' : None,
84 84 'pyerr' : None,
85 85 'stdout' : '',
86 86 'stderr' : '',
87 87 }
88 88 self.update(md)
89 89 self.update(dict(*args, **kwargs))
90 90
91 91 def __getattr__(self, key):
92 92 """getattr aliased to getitem"""
93 93 if key in self.iterkeys():
94 94 return self[key]
95 95 else:
96 96 raise AttributeError(key)
97 97
98 98 def __setattr__(self, key, value):
99 99 """setattr aliased to setitem, with strict"""
100 100 if key in self.iterkeys():
101 101 self[key] = value
102 102 else:
103 103 raise AttributeError(key)
104 104
105 105 def __setitem__(self, key, value):
106 106 """strict static key enforcement"""
107 107 if key in self.iterkeys():
108 108 dict.__setitem__(self, key, value)
109 109 else:
110 110 raise KeyError(key)
111 111
112 112
113 113 class Client(HasTraits):
114 114 """A semi-synchronous client to the IPython ZMQ cluster
115 115
116 116 Parameters
117 117 ----------
118 118
119 119 url_or_file : bytes; zmq url or path to ipcontroller-client.json
120 120 Connection information for the Hub's registration. If a json connector
121 121 file is given, then likely no further configuration is necessary.
122 122 [Default: use profile]
123 123 profile : bytes
124 124 The name of the Cluster profile to be used to find connector information.
125 125 [Default: 'default']
126 126 context : zmq.Context
127 127 Pass an existing zmq.Context instance, otherwise the client will create its own.
128 128 debug : bool
129 129 flag for lots of message printing for debug purposes
130 130 timeout : int/float
131 131 time (in seconds) to wait for connection replies from the Hub
132 132 [Default: 10]
133 133
134 134 #-------------- session related args ----------------
135 135
136 136 config : Config object
137 137 If specified, this will be relayed to the Session for configuration
138 138 username : str
139 139 set username for the session object
140 140 packer : str (import_string) or callable
141 141 Can be either the simple keyword 'json' or 'pickle', or an import_string to a
142 142 function to serialize messages. Must support same input as
143 143 JSON, and output must be bytes.
144 144 You can pass a callable directly as `pack`
145 145 unpacker : str (import_string) or callable
146 146 The inverse of packer. Only necessary if packer is specified as *not* one
147 147 of 'json' or 'pickle'.
148 148
149 149 #-------------- ssh related args ----------------
150 150 # These are args for configuring the ssh tunnel to be used
151 151 # credentials are used to forward connections over ssh to the Controller
152 152 # Note that the ip given in `addr` needs to be relative to sshserver
153 153 # The most basic case is to leave addr as pointing to localhost (127.0.0.1),
154 154 # and set sshserver as the same machine the Controller is on. However,
155 155 # the only requirement is that sshserver is able to see the Controller
156 156 # (i.e. is within the same trusted network).
157 157
158 158 sshserver : str
159 159 A string of the form passed to ssh, i.e. 'server.tld' or 'user@server.tld:port'
160 160 If keyfile or password is specified, and this is not, it will default to
161 161 the ip given in addr.
162 162 sshkey : str; path to public ssh key file
163 163 This specifies a key to be used in ssh login, default None.
164 164 Regular default ssh keys will be used without specifying this argument.
165 165 password : str
166 166 Your ssh password to sshserver. Note that if this is left None,
167 167 you will be prompted for it if passwordless key based login is unavailable.
168 168 paramiko : bool
169 169 flag for whether to use paramiko instead of shell ssh for tunneling.
170 170 [default: True on win32, False else]
171 171
172 172 ------- exec authentication args -------
173 173 If even localhost is untrusted, you can have some protection against
174 174 unauthorized execution by signing messages with HMAC digests.
175 175 Messages are still sent as cleartext, so if someone can snoop your
176 176 loopback traffic this will not protect your privacy, but will prevent
177 177 unauthorized execution.
178 178
179 179 exec_key : str
180 180 an authentication key or file containing a key
181 181 default: None
182 182
183 183
184 184 Attributes
185 185 ----------
186 186
187 187 ids : list of int engine IDs
188 188 requesting the ids attribute always synchronizes
189 189 the registration state. To request ids without synchronization,
190 190 use semi-private _ids attributes.
191 191
192 192 history : list of msg_ids
193 193 a list of msg_ids, keeping track of all the execution
194 194 messages you have submitted in order.
195 195
196 196 outstanding : set of msg_ids
197 197 a set of msg_ids that have been submitted, but whose
198 198 results have not yet been received.
199 199
200 200 results : dict
201 201 a dict of all our results, keyed by msg_id
202 202
203 203 block : bool
204 204 determines default behavior when block not specified
205 205 in execution methods
206 206
207 207 Methods
208 208 -------
209 209
210 210 spin
211 211 flushes incoming results and registration state changes
212 212 control methods spin, and requesting `ids` also ensures up to date
213 213
214 214 wait
215 215 wait on one or more msg_ids
216 216
217 217 execution methods
218 218 apply
219 219 legacy: execute, run
220 220
221 221 data movement
222 222 push, pull, scatter, gather
223 223
224 224 query methods
225 225 queue_status, get_result, purge, result_status
226 226
227 227 control methods
228 228 abort, shutdown
229 229
230 230 """
231 231
232 232
233 233 block = Bool(False)
234 234 outstanding = Set()
235 235 results = Instance('collections.defaultdict', (dict,))
236 236 metadata = Instance('collections.defaultdict', (Metadata,))
237 237 history = List()
238 238 debug = Bool(False)
239 239 profile=Unicode('default')
240 240
241 241 _outstanding_dict = Instance('collections.defaultdict', (set,))
242 242 _ids = List()
243 243 _connected=Bool(False)
244 244 _ssh=Bool(False)
245 245 _context = Instance('zmq.Context')
246 246 _config = Dict()
247 247 _engines=Instance(util.ReverseDict, (), {})
248 248 # _hub_socket=Instance('zmq.Socket')
249 249 _query_socket=Instance('zmq.Socket')
250 250 _control_socket=Instance('zmq.Socket')
251 251 _iopub_socket=Instance('zmq.Socket')
252 252 _notification_socket=Instance('zmq.Socket')
253 253 _mux_socket=Instance('zmq.Socket')
254 254 _task_socket=Instance('zmq.Socket')
255 255 _task_scheme=Unicode()
256 256 _closed = False
257 257 _ignored_control_replies=Int(0)
258 258 _ignored_hub_replies=Int(0)
259 259
260 260 def __init__(self, url_or_file=None, profile='default', profile_dir=None, ipython_dir=None,
261 261 context=None, debug=False, exec_key=None,
262 262 sshserver=None, sshkey=None, password=None, paramiko=None,
263 263 timeout=10, **extra_args
264 264 ):
265 265 super(Client, self).__init__(debug=debug, profile=profile)
266 266 if context is None:
267 267 context = zmq.Context.instance()
268 268 self._context = context
269 269
270 270
271 271 self._setup_profile_dir(profile, profile_dir, ipython_dir)
272 272 if self._cd is not None:
273 273 if url_or_file is None:
274 274 url_or_file = pjoin(self._cd.security_dir, 'ipcontroller-client.json')
275 275 assert url_or_file is not None, "I can't find enough information to connect to a hub!"\
276 276 " Please specify at least one of url_or_file or profile."
277 277
278 278 try:
279 279 util.validate_url(url_or_file)
280 280 except AssertionError:
281 281 if not os.path.exists(url_or_file):
282 282 if self._cd:
283 283 url_or_file = os.path.join(self._cd.security_dir, url_or_file)
284 284 assert os.path.exists(url_or_file), "Not a valid connection file or url: %r"%url_or_file
285 285 with open(url_or_file) as f:
286 286 cfg = json.loads(f.read())
287 287 else:
288 288 cfg = {'url':url_or_file}
289 289
290 290 # sync defaults from args, json:
291 291 if sshserver:
292 292 cfg['ssh'] = sshserver
293 293 if exec_key:
294 294 cfg['exec_key'] = exec_key
295 295 exec_key = cfg['exec_key']
296 296 sshserver=cfg['ssh']
297 297 url = cfg['url']
298 298 location = cfg.setdefault('location', None)
299 299 cfg['url'] = util.disambiguate_url(cfg['url'], location)
300 300 url = cfg['url']
301 301
302 302 self._config = cfg
303 303
304 304 self._ssh = bool(sshserver or sshkey or password)
305 305 if self._ssh and sshserver is None:
306 306 # default to ssh via localhost
307 307 sshserver = url.split('://')[1].split(':')[0]
308 308 if self._ssh and password is None:
309 309 if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko):
310 310 password=False
311 311 else:
312 312 password = getpass("SSH Password for %s: "%sshserver)
313 313 ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko)
314 314
315 315 # configure and construct the session
316 316 if exec_key is not None:
317 317 if os.path.isfile(exec_key):
318 318 extra_args['keyfile'] = exec_key
319 319 else:
320 320 extra_args['key'] = exec_key
321 321 self.session = Session(**extra_args)
322 322
323 323 self._query_socket = self._context.socket(zmq.XREQ)
324 324 self._query_socket.setsockopt(zmq.IDENTITY, self.session.session)
325 325 if self._ssh:
326 326 tunnel.tunnel_connection(self._query_socket, url, sshserver, **ssh_kwargs)
327 327 else:
328 328 self._query_socket.connect(url)
329 329
330 330 self.session.debug = self.debug
331 331
332 332 self._notification_handlers = {'registration_notification' : self._register_engine,
333 333 'unregistration_notification' : self._unregister_engine,
334 334 'shutdown_notification' : lambda msg: self.close(),
335 335 }
336 336 self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
337 337 'apply_reply' : self._handle_apply_reply}
338 338 self._connect(sshserver, ssh_kwargs, timeout)
339 339
340 340 def __del__(self):
341 341 """cleanup sockets, but _not_ context."""
342 342 self.close()
343 343
344 344 def _setup_profile_dir(self, profile, profile_dir, ipython_dir):
345 345 if ipython_dir is None:
346 346 ipython_dir = get_ipython_dir()
347 347 if profile_dir is not None:
348 348 try:
349 349 self._cd = ProfileDir.find_profile_dir(profile_dir)
350 350 return
351 351 except ProfileDirError:
352 352 pass
353 353 elif profile is not None:
354 354 try:
355 355 self._cd = ProfileDir.find_profile_dir_by_name(
356 356 ipython_dir, profile)
357 357 return
358 358 except ProfileDirError:
359 359 pass
360 360 self._cd = None
361 361
362 362 def _update_engines(self, engines):
363 363 """Update our engines dict and _ids from a dict of the form: {id:uuid}."""
364 364 for k,v in engines.iteritems():
365 365 eid = int(k)
366 366 self._engines[eid] = bytes(v) # force not unicode
367 367 self._ids.append(eid)
368 368 self._ids = sorted(self._ids)
369 369 if sorted(self._engines.keys()) != range(len(self._engines)) and \
370 370 self._task_scheme == 'pure' and self._task_socket:
371 371 self._stop_scheduling_tasks()
372 372
373 373 def _stop_scheduling_tasks(self):
374 374 """Stop scheduling tasks because an engine has been unregistered
375 375 from a pure ZMQ scheduler.
376 376 """
377 377 self._task_socket.close()
378 378 self._task_socket = None
379 379 msg = "An engine has been unregistered, and we are using pure " +\
380 380 "ZMQ task scheduling. Task farming will be disabled."
381 381 if self.outstanding:
382 382 msg += " If you were running tasks when this happened, " +\
383 383 "some `outstanding` msg_ids may never resolve."
384 384 warnings.warn(msg, RuntimeWarning)
385 385
386 386 def _build_targets(self, targets):
387 387 """Turn valid target IDs or 'all' into two lists:
388 388 (int_ids, uuids).
389 389 """
390 390 if not self._ids:
391 391 # flush notification socket if no engines yet, just in case
392 392 if not self.ids:
393 393 raise error.NoEnginesRegistered("Can't build targets without any engines")
394 394
395 395 if targets is None:
396 396 targets = self._ids
397 397 elif isinstance(targets, str):
398 398 if targets.lower() == 'all':
399 399 targets = self._ids
400 400 else:
401 401 raise TypeError("%r not valid str target, must be 'all'"%(targets))
402 402 elif isinstance(targets, int):
403 403 if targets < 0:
404 404 targets = self.ids[targets]
405 405 if targets not in self._ids:
406 406 raise IndexError("No such engine: %i"%targets)
407 407 targets = [targets]
408 408
409 409 if isinstance(targets, slice):
410 410 indices = range(len(self._ids))[targets]
411 411 ids = self.ids
412 412 targets = [ ids[i] for i in indices ]
413 413
414 414 if not isinstance(targets, (tuple, list, xrange)):
415 415 raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets)))
416 416
417 417 return [self._engines[t] for t in targets], list(targets)
418 418
419 419 def _connect(self, sshserver, ssh_kwargs, timeout):
420 420 """setup all our socket connections to the cluster. This is called from
421 421 __init__."""
422 422
423 423 # Maybe allow reconnecting?
424 424 if self._connected:
425 425 return
426 426 self._connected=True
427 427
428 428 def connect_socket(s, url):
429 429 url = util.disambiguate_url(url, self._config['location'])
430 430 if self._ssh:
431 431 return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
432 432 else:
433 433 return s.connect(url)
434 434
435 435 self.session.send(self._query_socket, 'connection_request')
436 436 r,w,x = zmq.select([self._query_socket],[],[], timeout)
437 437 if not r:
438 438 raise error.TimeoutError("Hub connection request timed out")
439 439 idents,msg = self.session.recv(self._query_socket,mode=0)
440 440 if self.debug:
441 441 pprint(msg)
442 442 msg = Message(msg)
443 443 content = msg.content
444 444 self._config['registration'] = dict(content)
445 445 if content.status == 'ok':
446 446 if content.mux:
447 447 self._mux_socket = self._context.socket(zmq.XREQ)
448 448 self._mux_socket.setsockopt(zmq.IDENTITY, self.session.session)
449 449 connect_socket(self._mux_socket, content.mux)
450 450 if content.task:
451 451 self._task_scheme, task_addr = content.task
452 452 self._task_socket = self._context.socket(zmq.XREQ)
453 453 self._task_socket.setsockopt(zmq.IDENTITY, self.session.session)
454 454 connect_socket(self._task_socket, task_addr)
455 455 if content.notification:
456 456 self._notification_socket = self._context.socket(zmq.SUB)
457 457 connect_socket(self._notification_socket, content.notification)
458 458 self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'')
459 459 # if content.query:
460 460 # self._query_socket = self._context.socket(zmq.XREQ)
461 461 # self._query_socket.setsockopt(zmq.IDENTITY, self.session.session)
462 462 # connect_socket(self._query_socket, content.query)
463 463 if content.control:
464 464 self._control_socket = self._context.socket(zmq.XREQ)
465 465 self._control_socket.setsockopt(zmq.IDENTITY, self.session.session)
466 466 connect_socket(self._control_socket, content.control)
467 467 if content.iopub:
468 468 self._iopub_socket = self._context.socket(zmq.SUB)
469 469 self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'')
470 470 self._iopub_socket.setsockopt(zmq.IDENTITY, self.session.session)
471 471 connect_socket(self._iopub_socket, content.iopub)
472 472 self._update_engines(dict(content.engines))
473 473 else:
474 474 self._connected = False
475 475 raise Exception("Failed to connect!")
476 476
477 477 #--------------------------------------------------------------------------
478 478 # handlers and callbacks for incoming messages
479 479 #--------------------------------------------------------------------------
480 480
481 481 def _unwrap_exception(self, content):
482 482 """unwrap exception, and remap engine_id to int."""
483 483 e = error.unwrap_exception(content)
484 484 # print e.traceback
485 485 if e.engine_info:
486 486 e_uuid = e.engine_info['engine_uuid']
487 487 eid = self._engines[e_uuid]
488 488 e.engine_info['engine_id'] = eid
489 489 return e
490 490
491 491 def _extract_metadata(self, header, parent, content):
492 492 md = {'msg_id' : parent['msg_id'],
493 493 'received' : datetime.now(),
494 494 'engine_uuid' : header.get('engine', None),
495 495 'follow' : parent.get('follow', []),
496 496 'after' : parent.get('after', []),
497 497 'status' : content['status'],
498 498 }
499 499
500 500 if md['engine_uuid'] is not None:
501 501 md['engine_id'] = self._engines.get(md['engine_uuid'], None)
502 502
503 503 if 'date' in parent:
504 504 md['submitted'] = parent['date']
505 505 if 'started' in header:
506 506 md['started'] = header['started']
507 507 if 'date' in header:
508 508 md['completed'] = header['date']
509 509 return md
510 510
511 511 def _register_engine(self, msg):
512 512 """Register a new engine, and update our connection info."""
513 513 content = msg['content']
514 514 eid = content['id']
515 515 d = {eid : content['queue']}
516 516 self._update_engines(d)
517 517
518 518 def _unregister_engine(self, msg):
519 519 """Unregister an engine that has died."""
520 520 content = msg['content']
521 521 eid = int(content['id'])
522 522 if eid in self._ids:
523 523 self._ids.remove(eid)
524 524 uuid = self._engines.pop(eid)
525 525
526 526 self._handle_stranded_msgs(eid, uuid)
527 527
528 528 if self._task_socket and self._task_scheme == 'pure':
529 529 self._stop_scheduling_tasks()
530 530
531 531 def _handle_stranded_msgs(self, eid, uuid):
532 532 """Handle messages known to be on an engine when the engine unregisters.
533 533
534 534 It is possible that this will fire prematurely - that is, an engine will
535 535 go down after completing a result, and the client will be notified
536 536 of the unregistration and later receive the successful result.
537 537 """
538 538
539 539 outstanding = self._outstanding_dict[uuid]
540 540
541 541 for msg_id in list(outstanding):
542 542 if msg_id in self.results:
543 543 # we already
544 544 continue
545 545 try:
546 546 raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id))
547 547 except:
548 548 content = error.wrap_exception()
549 549 # build a fake message:
550 550 parent = {}
551 551 header = {}
552 552 parent['msg_id'] = msg_id
553 553 header['engine'] = uuid
554 554 header['date'] = datetime.now()
555 555 msg = dict(parent_header=parent, header=header, content=content)
556 556 self._handle_apply_reply(msg)
557 557
558 558 def _handle_execute_reply(self, msg):
559 559 """Save the reply to an execute_request into our results.
560 560
561 561 execute messages are never actually used. apply is used instead.
562 562 """
563 563
564 564 parent = msg['parent_header']
565 565 msg_id = parent['msg_id']
566 566 if msg_id not in self.outstanding:
567 567 if msg_id in self.history:
568 568 print ("got stale result: %s"%msg_id)
569 569 else:
570 570 print ("got unknown result: %s"%msg_id)
571 571 else:
572 572 self.outstanding.remove(msg_id)
573 573 self.results[msg_id] = self._unwrap_exception(msg['content'])
574 574
575 575 def _handle_apply_reply(self, msg):
576 576 """Save the reply to an apply_request into our results."""
577 577 parent = msg['parent_header']
578 578 msg_id = parent['msg_id']
579 579 if msg_id not in self.outstanding:
580 580 if msg_id in self.history:
581 581 print ("got stale result: %s"%msg_id)
582 582 print self.results[msg_id]
583 583 print msg
584 584 else:
585 585 print ("got unknown result: %s"%msg_id)
586 586 else:
587 587 self.outstanding.remove(msg_id)
588 588 content = msg['content']
589 589 header = msg['header']
590 590
591 591 # construct metadata:
592 592 md = self.metadata[msg_id]
593 593 md.update(self._extract_metadata(header, parent, content))
594 594 # is this redundant?
595 595 self.metadata[msg_id] = md
596 596
597 597 e_outstanding = self._outstanding_dict[md['engine_uuid']]
598 598 if msg_id in e_outstanding:
599 599 e_outstanding.remove(msg_id)
600 600
601 601 # construct result:
602 602 if content['status'] == 'ok':
603 603 self.results[msg_id] = util.unserialize_object(msg['buffers'])[0]
604 604 elif content['status'] == 'aborted':
605 605 self.results[msg_id] = error.TaskAborted(msg_id)
606 606 elif content['status'] == 'resubmitted':
607 607 # TODO: handle resubmission
608 608 pass
609 609 else:
610 610 self.results[msg_id] = self._unwrap_exception(content)
611 611
612 612 def _flush_notifications(self):
613 613 """Flush notifications of engine registrations waiting
614 614 in ZMQ queue."""
615 615 idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
616 616 while msg is not None:
617 617 if self.debug:
618 618 pprint(msg)
619 619 msg_type = msg['msg_type']
620 620 handler = self._notification_handlers.get(msg_type, None)
621 621 if handler is None:
622 622 raise Exception("Unhandled message type: %s"%msg.msg_type)
623 623 else:
624 624 handler(msg)
625 625 idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK)
626 626
627 627 def _flush_results(self, sock):
628 628 """Flush task or queue results waiting in ZMQ queue."""
629 629 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
630 630 while msg is not None:
631 631 if self.debug:
632 632 pprint(msg)
633 633 msg_type = msg['msg_type']
634 634 handler = self._queue_handlers.get(msg_type, None)
635 635 if handler is None:
636 636 raise Exception("Unhandled message type: %s"%msg.msg_type)
637 637 else:
638 638 handler(msg)
639 639 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
640 640
641 641 def _flush_control(self, sock):
642 642 """Flush replies from the control channel waiting
643 643 in the ZMQ queue.
644 644
645 645 Currently: ignore them."""
646 646 if self._ignored_control_replies <= 0:
647 647 return
648 648 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
649 649 while msg is not None:
650 650 self._ignored_control_replies -= 1
651 651 if self.debug:
652 652 pprint(msg)
653 653 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
654 654
655 655 def _flush_ignored_control(self):
656 656 """flush ignored control replies"""
657 657 while self._ignored_control_replies > 0:
658 658 self.session.recv(self._control_socket)
659 659 self._ignored_control_replies -= 1
660 660
661 661 def _flush_ignored_hub_replies(self):
662 662 ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
663 663 while msg is not None:
664 664 ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK)
665 665
666 666 def _flush_iopub(self, sock):
667 667 """Flush replies from the iopub channel waiting
668 668 in the ZMQ queue.
669 669 """
670 670 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
671 671 while msg is not None:
672 672 if self.debug:
673 673 pprint(msg)
674 674 parent = msg['parent_header']
675 675 msg_id = parent['msg_id']
676 676 content = msg['content']
677 677 header = msg['header']
678 678 msg_type = msg['msg_type']
679 679
680 680 # init metadata:
681 681 md = self.metadata[msg_id]
682 682
683 683 if msg_type == 'stream':
684 684 name = content['name']
685 685 s = md[name] or ''
686 686 md[name] = s + content['data']
687 687 elif msg_type == 'pyerr':
688 688 md.update({'pyerr' : self._unwrap_exception(content)})
689 689 elif msg_type == 'pyin':
690 690 md.update({'pyin' : content['code']})
691 691 else:
692 692 md.update({msg_type : content.get('data', '')})
693 693
694 694 # reduntant?
695 695 self.metadata[msg_id] = md
696 696
697 697 idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK)
698 698
699 699 #--------------------------------------------------------------------------
700 700 # len, getitem
701 701 #--------------------------------------------------------------------------
702 702
703 703 def __len__(self):
704 704 """len(client) returns # of engines."""
705 705 return len(self.ids)
706 706
707 707 def __getitem__(self, key):
708 708 """index access returns DirectView multiplexer objects
709 709
710 710 Must be int, slice, or list/tuple/xrange of ints"""
711 711 if not isinstance(key, (int, slice, tuple, list, xrange)):
712 712 raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key)))
713 713 else:
714 714 return self.direct_view(key)
715 715
716 716 #--------------------------------------------------------------------------
717 717 # Begin public methods
718 718 #--------------------------------------------------------------------------
719 719
720 720 @property
721 721 def ids(self):
722 722 """Always up-to-date ids property."""
723 723 self._flush_notifications()
724 724 # always copy:
725 725 return list(self._ids)
726 726
727 727 def close(self):
728 728 if self._closed:
729 729 return
730 730 snames = filter(lambda n: n.endswith('socket'), dir(self))
731 731 for socket in map(lambda name: getattr(self, name), snames):
732 732 if isinstance(socket, zmq.Socket) and not socket.closed:
733 733 socket.close()
734 734 self._closed = True
735 735
736 736 def spin(self):
737 737 """Flush any registration notifications and execution results
738 738 waiting in the ZMQ queue.
739 739 """
740 740 if self._notification_socket:
741 741 self._flush_notifications()
742 742 if self._mux_socket:
743 743 self._flush_results(self._mux_socket)
744 744 if self._task_socket:
745 745 self._flush_results(self._task_socket)
746 746 if self._control_socket:
747 747 self._flush_control(self._control_socket)
748 748 if self._iopub_socket:
749 749 self._flush_iopub(self._iopub_socket)
750 750 if self._query_socket:
751 751 self._flush_ignored_hub_replies()
752 752
753 753 def wait(self, jobs=None, timeout=-1):
754 754 """waits on one or more `jobs`, for up to `timeout` seconds.
755 755
756 756 Parameters
757 757 ----------
758 758
759 759 jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
760 760 ints are indices to self.history
761 761 strs are msg_ids
762 762 default: wait on all outstanding messages
763 763 timeout : float
764 764 a time in seconds, after which to give up.
765 765 default is -1, which means no timeout
766 766
767 767 Returns
768 768 -------
769 769
770 770 True : when all msg_ids are done
771 771 False : timeout reached, some msg_ids still outstanding
772 772 """
773 773 tic = time.time()
774 774 if jobs is None:
775 775 theids = self.outstanding
776 776 else:
777 777 if isinstance(jobs, (int, str, AsyncResult)):
778 778 jobs = [jobs]
779 779 theids = set()
780 780 for job in jobs:
781 781 if isinstance(job, int):
782 782 # index access
783 783 job = self.history[job]
784 784 elif isinstance(job, AsyncResult):
785 785 map(theids.add, job.msg_ids)
786 786 continue
787 787 theids.add(job)
788 788 if not theids.intersection(self.outstanding):
789 789 return True
790 790 self.spin()
791 791 while theids.intersection(self.outstanding):
792 792 if timeout >= 0 and ( time.time()-tic ) > timeout:
793 793 break
794 794 time.sleep(1e-3)
795 795 self.spin()
796 796 return len(theids.intersection(self.outstanding)) == 0
797 797
798 798 #--------------------------------------------------------------------------
799 799 # Control methods
800 800 #--------------------------------------------------------------------------
801 801
802 802 @spin_first
803 803 def clear(self, targets=None, block=None):
804 804 """Clear the namespace in target(s)."""
805 805 block = self.block if block is None else block
806 806 targets = self._build_targets(targets)[0]
807 807 for t in targets:
808 808 self.session.send(self._control_socket, 'clear_request', content={}, ident=t)
809 809 error = False
810 810 if block:
811 811 self._flush_ignored_control()
812 812 for i in range(len(targets)):
813 813 idents,msg = self.session.recv(self._control_socket,0)
814 814 if self.debug:
815 815 pprint(msg)
816 816 if msg['content']['status'] != 'ok':
817 817 error = self._unwrap_exception(msg['content'])
818 818 else:
819 819 self._ignored_control_replies += len(targets)
820 820 if error:
821 821 raise error
822 822
823 823
824 824 @spin_first
825 825 def abort(self, jobs=None, targets=None, block=None):
826 826 """Abort specific jobs from the execution queues of target(s).
827 827
828 828 This is a mechanism to prevent jobs that have already been submitted
829 829 from executing.
830 830
831 831 Parameters
832 832 ----------
833 833
834 834 jobs : msg_id, list of msg_ids, or AsyncResult
835 835 The jobs to be aborted
836 836
837 837
838 838 """
839 839 block = self.block if block is None else block
840 840 targets = self._build_targets(targets)[0]
841 841 msg_ids = []
842 842 if isinstance(jobs, (basestring,AsyncResult)):
843 843 jobs = [jobs]
844 844 bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)
845 845 if bad_ids:
846 846 raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
847 847 for j in jobs:
848 848 if isinstance(j, AsyncResult):
849 849 msg_ids.extend(j.msg_ids)
850 850 else:
851 851 msg_ids.append(j)
852 852 content = dict(msg_ids=msg_ids)
853 853 for t in targets:
854 854 self.session.send(self._control_socket, 'abort_request',
855 855 content=content, ident=t)
856 856 error = False
857 857 if block:
858 858 self._flush_ignored_control()
859 859 for i in range(len(targets)):
860 860 idents,msg = self.session.recv(self._control_socket,0)
861 861 if self.debug:
862 862 pprint(msg)
863 863 if msg['content']['status'] != 'ok':
864 864 error = self._unwrap_exception(msg['content'])
865 865 else:
866 866 self._ignored_control_replies += len(targets)
867 867 if error:
868 868 raise error
869 869
870 870 @spin_first
871 871 def shutdown(self, targets=None, restart=False, hub=False, block=None):
872 872 """Terminates one or more engine processes, optionally including the hub."""
873 873 block = self.block if block is None else block
874 874 if hub:
875 875 targets = 'all'
876 876 targets = self._build_targets(targets)[0]
877 877 for t in targets:
878 878 self.session.send(self._control_socket, 'shutdown_request',
879 879 content={'restart':restart},ident=t)
880 880 error = False
881 881 if block or hub:
882 882 self._flush_ignored_control()
883 883 for i in range(len(targets)):
884 884 idents,msg = self.session.recv(self._control_socket, 0)
885 885 if self.debug:
886 886 pprint(msg)
887 887 if msg['content']['status'] != 'ok':
888 888 error = self._unwrap_exception(msg['content'])
889 889 else:
890 890 self._ignored_control_replies += len(targets)
891 891
892 892 if hub:
893 893 time.sleep(0.25)
894 894 self.session.send(self._query_socket, 'shutdown_request')
895 895 idents,msg = self.session.recv(self._query_socket, 0)
896 896 if self.debug:
897 897 pprint(msg)
898 898 if msg['content']['status'] != 'ok':
899 899 error = self._unwrap_exception(msg['content'])
900 900
901 901 if error:
902 902 raise error
903 903
904 904 #--------------------------------------------------------------------------
905 905 # Execution related methods
906 906 #--------------------------------------------------------------------------
907 907
908 908 def _maybe_raise(self, result):
909 909 """wrapper for maybe raising an exception if apply failed."""
910 910 if isinstance(result, error.RemoteError):
911 911 raise result
912 912
913 913 return result
914 914
915 915 def send_apply_message(self, socket, f, args=None, kwargs=None, subheader=None, track=False,
916 916 ident=None):
917 917 """construct and send an apply message via a socket.
918 918
919 919 This is the principal method with which all engine execution is performed by views.
920 920 """
921 921
922 922 assert not self._closed, "cannot use me anymore, I'm closed!"
923 923 # defaults:
924 924 args = args if args is not None else []
925 925 kwargs = kwargs if kwargs is not None else {}
926 926 subheader = subheader if subheader is not None else {}
927 927
928 928 # validate arguments
929 929 if not callable(f):
930 930 raise TypeError("f must be callable, not %s"%type(f))
931 931 if not isinstance(args, (tuple, list)):
932 932 raise TypeError("args must be tuple or list, not %s"%type(args))
933 933 if not isinstance(kwargs, dict):
934 934 raise TypeError("kwargs must be dict, not %s"%type(kwargs))
935 935 if not isinstance(subheader, dict):
936 936 raise TypeError("subheader must be dict, not %s"%type(subheader))
937 937
938 938 bufs = util.pack_apply_message(f,args,kwargs)
939 939
940 940 msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident,
941 941 subheader=subheader, track=track)
942 942
943 943 msg_id = msg['msg_id']
944 944 self.outstanding.add(msg_id)
945 945 if ident:
946 946 # possibly routed to a specific engine
947 947 if isinstance(ident, list):
948 948 ident = ident[-1]
949 949 if ident in self._engines.values():
950 950 # save for later, in case of engine death
951 951 self._outstanding_dict[ident].add(msg_id)
952 952 self.history.append(msg_id)
953 953 self.metadata[msg_id]['submitted'] = datetime.now()
954 954
955 955 return msg
956 956
957 957 #--------------------------------------------------------------------------
958 958 # construct a View object
959 959 #--------------------------------------------------------------------------
960 960
961 961 def load_balanced_view(self, targets=None):
962 962 """construct a DirectView object.
963 963
964 964 If no arguments are specified, create a LoadBalancedView
965 965 using all engines.
966 966
967 967 Parameters
968 968 ----------
969 969
970 970 targets: list,slice,int,etc. [default: use all engines]
971 971 The subset of engines across which to load-balance
972 972 """
973 973 if targets is not None:
974 974 targets = self._build_targets(targets)[1]
975 975 return LoadBalancedView(client=self, socket=self._task_socket, targets=targets)
976 976
977 977 def direct_view(self, targets='all'):
978 978 """construct a DirectView object.
979 979
980 980 If no targets are specified, create a DirectView
981 981 using all engines.
982 982
983 983 Parameters
984 984 ----------
985 985
986 986 targets: list,slice,int,etc. [default: use all engines]
987 987 The engines to use for the View
988 988 """
989 989 single = isinstance(targets, int)
990 990 targets = self._build_targets(targets)[1]
991 991 if single:
992 992 targets = targets[0]
993 993 return DirectView(client=self, socket=self._mux_socket, targets=targets)
994 994
995 995 #--------------------------------------------------------------------------
996 996 # Query methods
997 997 #--------------------------------------------------------------------------
998 998
999 999 @spin_first
1000 1000 def get_result(self, indices_or_msg_ids=None, block=None):
1001 1001 """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object.
1002 1002
1003 1003 If the client already has the results, no request to the Hub will be made.
1004 1004
1005 1005 This is a convenient way to construct AsyncResult objects, which are wrappers
1006 1006 that include metadata about execution, and allow for awaiting results that
1007 1007 were not submitted by this Client.
1008 1008
1009 1009 It can also be a convenient way to retrieve the metadata associated with
1010 1010 blocking execution, since it always retrieves
1011 1011
1012 1012 Examples
1013 1013 --------
1014 1014 ::
1015 1015
1016 1016 In [10]: r = client.apply()
1017 1017
1018 1018 Parameters
1019 1019 ----------
1020 1020
1021 1021 indices_or_msg_ids : integer history index, str msg_id, or list of either
1022 1022 The indices or msg_ids of indices to be retrieved
1023 1023
1024 1024 block : bool
1025 1025 Whether to wait for the result to be done
1026 1026
1027 1027 Returns
1028 1028 -------
1029 1029
1030 1030 AsyncResult
1031 1031 A single AsyncResult object will always be returned.
1032 1032
1033 1033 AsyncHubResult
1034 1034 A subclass of AsyncResult that retrieves results from the Hub
1035 1035
1036 1036 """
1037 1037 block = self.block if block is None else block
1038 1038 if indices_or_msg_ids is None:
1039 1039 indices_or_msg_ids = -1
1040 1040
1041 1041 if not isinstance(indices_or_msg_ids, (list,tuple)):
1042 1042 indices_or_msg_ids = [indices_or_msg_ids]
1043 1043
1044 1044 theids = []
1045 1045 for id in indices_or_msg_ids:
1046 1046 if isinstance(id, int):
1047 1047 id = self.history[id]
1048 1048 if not isinstance(id, str):
1049 1049 raise TypeError("indices must be str or int, not %r"%id)
1050 1050 theids.append(id)
1051 1051
1052 1052 local_ids = filter(lambda msg_id: msg_id in self.history or msg_id in self.results, theids)
1053 1053 remote_ids = filter(lambda msg_id: msg_id not in local_ids, theids)
1054 1054
1055 1055 if remote_ids:
1056 1056 ar = AsyncHubResult(self, msg_ids=theids)
1057 1057 else:
1058 1058 ar = AsyncResult(self, msg_ids=theids)
1059 1059
1060 1060 if block:
1061 1061 ar.wait()
1062 1062
1063 1063 return ar
1064 1064
1065 1065 @spin_first
1066 1066 def resubmit(self, indices_or_msg_ids=None, subheader=None, block=None):
1067 1067 """Resubmit one or more tasks.
1068 1068
1069 1069 in-flight tasks may not be resubmitted.
1070 1070
1071 1071 Parameters
1072 1072 ----------
1073 1073
1074 1074 indices_or_msg_ids : integer history index, str msg_id, or list of either
1075 1075 The indices or msg_ids of indices to be retrieved
1076 1076
1077 1077 block : bool
1078 1078 Whether to wait for the result to be done
1079 1079
1080 1080 Returns
1081 1081 -------
1082 1082
1083 1083 AsyncHubResult
1084 1084 A subclass of AsyncResult that retrieves results from the Hub
1085 1085
1086 1086 """
1087 1087 block = self.block if block is None else block
1088 1088 if indices_or_msg_ids is None:
1089 1089 indices_or_msg_ids = -1
1090 1090
1091 1091 if not isinstance(indices_or_msg_ids, (list,tuple)):
1092 1092 indices_or_msg_ids = [indices_or_msg_ids]
1093 1093
1094 1094 theids = []
1095 1095 for id in indices_or_msg_ids:
1096 1096 if isinstance(id, int):
1097 1097 id = self.history[id]
1098 1098 if not isinstance(id, str):
1099 1099 raise TypeError("indices must be str or int, not %r"%id)
1100 1100 theids.append(id)
1101 1101
1102 1102 for msg_id in theids:
1103 1103 self.outstanding.discard(msg_id)
1104 1104 if msg_id in self.history:
1105 1105 self.history.remove(msg_id)
1106 1106 self.results.pop(msg_id, None)
1107 1107 self.metadata.pop(msg_id, None)
1108 1108 content = dict(msg_ids = theids)
1109 1109
1110 1110 self.session.send(self._query_socket, 'resubmit_request', content)
1111 1111
1112 1112 zmq.select([self._query_socket], [], [])
1113 1113 idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
1114 1114 if self.debug:
1115 1115 pprint(msg)
1116 1116 content = msg['content']
1117 1117 if content['status'] != 'ok':
1118 1118 raise self._unwrap_exception(content)
1119 1119
1120 1120 ar = AsyncHubResult(self, msg_ids=theids)
1121 1121
1122 1122 if block:
1123 1123 ar.wait()
1124 1124
1125 1125 return ar
1126 1126
1127 1127 @spin_first
1128 1128 def result_status(self, msg_ids, status_only=True):
1129 1129 """Check on the status of the result(s) of the apply request with `msg_ids`.
1130 1130
1131 1131 If status_only is False, then the actual results will be retrieved, else
1132 1132 only the status of the results will be checked.
1133 1133
1134 1134 Parameters
1135 1135 ----------
1136 1136
1137 1137 msg_ids : list of msg_ids
1138 1138 if int:
1139 1139 Passed as index to self.history for convenience.
1140 1140 status_only : bool (default: True)
1141 1141 if False:
1142 1142 Retrieve the actual results of completed tasks.
1143 1143
1144 1144 Returns
1145 1145 -------
1146 1146
1147 1147 results : dict
1148 1148 There will always be the keys 'pending' and 'completed', which will
1149 1149 be lists of msg_ids that are incomplete or complete. If `status_only`
1150 1150 is False, then completed results will be keyed by their `msg_id`.
1151 1151 """
1152 1152 if not isinstance(msg_ids, (list,tuple)):
1153 1153 msg_ids = [msg_ids]
1154 1154
1155 1155 theids = []
1156 1156 for msg_id in msg_ids:
1157 1157 if isinstance(msg_id, int):
1158 1158 msg_id = self.history[msg_id]
1159 1159 if not isinstance(msg_id, basestring):
1160 1160 raise TypeError("msg_ids must be str, not %r"%msg_id)
1161 1161 theids.append(msg_id)
1162 1162
1163 1163 completed = []
1164 1164 local_results = {}
1165 1165
1166 1166 # comment this block out to temporarily disable local shortcut:
1167 1167 for msg_id in theids:
1168 1168 if msg_id in self.results:
1169 1169 completed.append(msg_id)
1170 1170 local_results[msg_id] = self.results[msg_id]
1171 1171 theids.remove(msg_id)
1172 1172
1173 1173 if theids: # some not locally cached
1174 1174 content = dict(msg_ids=theids, status_only=status_only)
1175 1175 msg = self.session.send(self._query_socket, "result_request", content=content)
1176 1176 zmq.select([self._query_socket], [], [])
1177 1177 idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
1178 1178 if self.debug:
1179 1179 pprint(msg)
1180 1180 content = msg['content']
1181 1181 if content['status'] != 'ok':
1182 1182 raise self._unwrap_exception(content)
1183 1183 buffers = msg['buffers']
1184 1184 else:
1185 1185 content = dict(completed=[],pending=[])
1186 1186
1187 1187 content['completed'].extend(completed)
1188 1188
1189 1189 if status_only:
1190 1190 return content
1191 1191
1192 1192 failures = []
1193 1193 # load cached results into result:
1194 1194 content.update(local_results)
1195 1195
1196 1196 # update cache with results:
1197 1197 for msg_id in sorted(theids):
1198 1198 if msg_id in content['completed']:
1199 1199 rec = content[msg_id]
1200 1200 parent = rec['header']
1201 1201 header = rec['result_header']
1202 1202 rcontent = rec['result_content']
1203 1203 iodict = rec['io']
1204 1204 if isinstance(rcontent, str):
1205 1205 rcontent = self.session.unpack(rcontent)
1206 1206
1207 1207 md = self.metadata[msg_id]
1208 1208 md.update(self._extract_metadata(header, parent, rcontent))
1209 1209 md.update(iodict)
1210 1210
1211 1211 if rcontent['status'] == 'ok':
1212 1212 res,buffers = util.unserialize_object(buffers)
1213 1213 else:
1214 1214 print rcontent
1215 1215 res = self._unwrap_exception(rcontent)
1216 1216 failures.append(res)
1217 1217
1218 1218 self.results[msg_id] = res
1219 1219 content[msg_id] = res
1220 1220
1221 1221 if len(theids) == 1 and failures:
1222 1222 raise failures[0]
1223 1223
1224 1224 error.collect_exceptions(failures, "result_status")
1225 1225 return content
1226 1226
1227 1227 @spin_first
1228 1228 def queue_status(self, targets='all', verbose=False):
1229 1229 """Fetch the status of engine queues.
1230 1230
1231 1231 Parameters
1232 1232 ----------
1233 1233
1234 1234 targets : int/str/list of ints/strs
1235 1235 the engines whose states are to be queried.
1236 1236 default : all
1237 1237 verbose : bool
1238 1238 Whether to return lengths only, or lists of ids for each element
1239 1239 """
1240 1240 engine_ids = self._build_targets(targets)[1]
1241 1241 content = dict(targets=engine_ids, verbose=verbose)
1242 1242 self.session.send(self._query_socket, "queue_request", content=content)
1243 1243 idents,msg = self.session.recv(self._query_socket, 0)
1244 1244 if self.debug:
1245 1245 pprint(msg)
1246 1246 content = msg['content']
1247 1247 status = content.pop('status')
1248 1248 if status != 'ok':
1249 1249 raise self._unwrap_exception(content)
1250 1250 content = util.rekey(content)
1251 1251 if isinstance(targets, int):
1252 1252 return content[targets]
1253 1253 else:
1254 1254 return content
1255 1255
1256 1256 @spin_first
1257 1257 def purge_results(self, jobs=[], targets=[]):
1258 1258 """Tell the Hub to forget results.
1259 1259
1260 1260 Individual results can be purged by msg_id, or the entire
1261 1261 history of specific targets can be purged.
1262 1262
1263 1263 Parameters
1264 1264 ----------
1265 1265
1266 1266 jobs : str or list of str or AsyncResult objects
1267 1267 the msg_ids whose results should be forgotten.
1268 1268 targets : int/str/list of ints/strs
1269 1269 The targets, by uuid or int_id, whose entire history is to be purged.
1270 1270 Use `targets='all'` to scrub everything from the Hub's memory.
1271 1271
1272 1272 default : None
1273 1273 """
1274 1274 if not targets and not jobs:
1275 1275 raise ValueError("Must specify at least one of `targets` and `jobs`")
1276 1276 if targets:
1277 1277 targets = self._build_targets(targets)[1]
1278 1278
1279 1279 # construct msg_ids from jobs
1280 1280 msg_ids = []
1281 1281 if isinstance(jobs, (basestring,AsyncResult)):
1282 1282 jobs = [jobs]
1283 1283 bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs)
1284 1284 if bad_ids:
1285 1285 raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0])
1286 1286 for j in jobs:
1287 1287 if isinstance(j, AsyncResult):
1288 1288 msg_ids.extend(j.msg_ids)
1289 1289 else:
1290 1290 msg_ids.append(j)
1291 1291
1292 1292 content = dict(targets=targets, msg_ids=msg_ids)
1293 1293 self.session.send(self._query_socket, "purge_request", content=content)
1294 1294 idents, msg = self.session.recv(self._query_socket, 0)
1295 1295 if self.debug:
1296 1296 pprint(msg)
1297 1297 content = msg['content']
1298 1298 if content['status'] != 'ok':
1299 1299 raise self._unwrap_exception(content)
1300 1300
1301 1301 @spin_first
1302 1302 def hub_history(self):
1303 1303 """Get the Hub's history
1304 1304
1305 1305 Just like the Client, the Hub has a history, which is a list of msg_ids.
1306 1306 This will contain the history of all clients, and, depending on configuration,
1307 1307 may contain history across multiple cluster sessions.
1308 1308
1309 1309 Any msg_id returned here is a valid argument to `get_result`.
1310 1310
1311 1311 Returns
1312 1312 -------
1313 1313
1314 1314 msg_ids : list of strs
1315 1315 list of all msg_ids, ordered by task submission time.
1316 1316 """
1317 1317
1318 1318 self.session.send(self._query_socket, "history_request", content={})
1319 1319 idents, msg = self.session.recv(self._query_socket, 0)
1320 1320
1321 1321 if self.debug:
1322 1322 pprint(msg)
1323 1323 content = msg['content']
1324 1324 if content['status'] != 'ok':
1325 1325 raise self._unwrap_exception(content)
1326 1326 else:
1327 1327 return content['history']
1328 1328
1329 1329 @spin_first
1330 1330 def db_query(self, query, keys=None):
1331 1331 """Query the Hub's TaskRecord database
1332 1332
1333 1333 This will return a list of task record dicts that match `query`
1334 1334
1335 1335 Parameters
1336 1336 ----------
1337 1337
1338 1338 query : mongodb query dict
1339 1339 The search dict. See mongodb query docs for details.
1340 1340 keys : list of strs [optional]
1341 1341 The subset of keys to be returned. The default is to fetch everything but buffers.
1342 1342 'msg_id' will *always* be included.
1343 1343 """
1344 1344 if isinstance(keys, basestring):
1345 1345 keys = [keys]
1346 1346 content = dict(query=query, keys=keys)
1347 1347 self.session.send(self._query_socket, "db_request", content=content)
1348 1348 idents, msg = self.session.recv(self._query_socket, 0)
1349 1349 if self.debug:
1350 1350 pprint(msg)
1351 1351 content = msg['content']
1352 1352 if content['status'] != 'ok':
1353 1353 raise self._unwrap_exception(content)
1354 1354
1355 1355 records = content['records']
1356 1356
1357 1357 buffer_lens = content['buffer_lens']
1358 1358 result_buffer_lens = content['result_buffer_lens']
1359 1359 buffers = msg['buffers']
1360 1360 has_bufs = buffer_lens is not None
1361 1361 has_rbufs = result_buffer_lens is not None
1362 1362 for i,rec in enumerate(records):
1363 1363 # relink buffers
1364 1364 if has_bufs:
1365 1365 blen = buffer_lens[i]
1366 1366 rec['buffers'], buffers = buffers[:blen],buffers[blen:]
1367 1367 if has_rbufs:
1368 1368 blen = result_buffer_lens[i]
1369 1369 rec['result_buffers'], buffers = buffers[:blen],buffers[blen:]
1370 1370
1371 1371 return records
1372 1372
1373 1373 __all__ = [ 'Client' ]
@@ -1,391 +1,391 b''
1 1 """A TaskRecord backend using sqlite3
2 2
3 3 Authors:
4 4
5 5 * Min RK
6 6 """
7 7 #-----------------------------------------------------------------------------
8 8 # Copyright (C) 2011 The IPython Development Team
9 9 #
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 import json
15 15 import os
16 16 import cPickle as pickle
17 17 from datetime import datetime
18 18
19 19 import sqlite3
20 20
21 21 from zmq.eventloop import ioloop
22 22
23 23 from IPython.utils.traitlets import Unicode, Instance, List, Dict
24 24 from .dictdb import BaseDB
25 25 from IPython.utils.jsonutil import date_default, extract_dates, squash_dates
26 26
27 27 #-----------------------------------------------------------------------------
28 28 # SQLite operators, adapters, and converters
29 29 #-----------------------------------------------------------------------------
30 30
31 31 operators = {
32 32 '$lt' : "<",
33 33 '$gt' : ">",
34 34 # null is handled weird with ==,!=
35 35 '$eq' : "=",
36 36 '$ne' : "!=",
37 37 '$lte': "<=",
38 38 '$gte': ">=",
39 39 '$in' : ('=', ' OR '),
40 40 '$nin': ('!=', ' AND '),
41 41 # '$all': None,
42 42 # '$mod': None,
43 43 # '$exists' : None
44 44 }
45 45 null_operators = {
46 46 '=' : "IS NULL",
47 47 '!=' : "IS NOT NULL",
48 48 }
49 49
50 50 def _adapt_dict(d):
51 51 return json.dumps(d, default=date_default)
52 52
53 53 def _convert_dict(ds):
54 54 if ds is None:
55 55 return ds
56 56 else:
57 57 return extract_dates(json.loads(ds))
58 58
59 59 def _adapt_bufs(bufs):
60 60 # this is *horrible*
61 61 # copy buffers into single list and pickle it:
62 62 if bufs and isinstance(bufs[0], (bytes, buffer)):
63 63 return sqlite3.Binary(pickle.dumps(map(bytes, bufs),-1))
64 64 elif bufs:
65 65 return bufs
66 66 else:
67 67 return None
68 68
69 69 def _convert_bufs(bs):
70 70 if bs is None:
71 71 return []
72 72 else:
73 73 return pickle.loads(bytes(bs))
74 74
75 75 #-----------------------------------------------------------------------------
76 76 # SQLiteDB class
77 77 #-----------------------------------------------------------------------------
78 78
79 79 class SQLiteDB(BaseDB):
80 80 """SQLite3 TaskRecord backend."""
81 81
82 82 filename = Unicode('tasks.db', config=True,
83 83 help="""The filename of the sqlite task database. [default: 'tasks.db']""")
84 84 location = Unicode('', config=True,
85 85 help="""The directory containing the sqlite task database. The default
86 86 is to use the cluster_dir location.""")
87 87 table = Unicode("", config=True,
88 88 help="""The SQLite Table to use for storing tasks for this session. If unspecified,
89 89 a new table will be created with the Hub's IDENT. Specifying the table will result
90 90 in tasks from previous sessions being available via Clients' db_query and
91 91 get_result methods.""")
92 92
93 93 _db = Instance('sqlite3.Connection')
94 94 # the ordered list of column names
95 95 _keys = List(['msg_id' ,
96 96 'header' ,
97 97 'content',
98 98 'buffers',
99 99 'submitted',
100 100 'client_uuid' ,
101 101 'engine_uuid' ,
102 102 'started',
103 103 'completed',
104 104 'resubmitted',
105 105 'result_header' ,
106 106 'result_content' ,
107 107 'result_buffers' ,
108 108 'queue' ,
109 109 'pyin' ,
110 110 'pyout',
111 111 'pyerr',
112 112 'stdout',
113 113 'stderr',
114 114 ])
115 115 # sqlite datatypes for checking that db is current format
116 116 _types = Dict({'msg_id' : 'text' ,
117 117 'header' : 'dict text',
118 118 'content' : 'dict text',
119 119 'buffers' : 'bufs blob',
120 120 'submitted' : 'timestamp',
121 121 'client_uuid' : 'text',
122 122 'engine_uuid' : 'text',
123 123 'started' : 'timestamp',
124 124 'completed' : 'timestamp',
125 125 'resubmitted' : 'timestamp',
126 126 'result_header' : 'dict text',
127 127 'result_content' : 'dict text',
128 128 'result_buffers' : 'bufs blob',
129 129 'queue' : 'text',
130 130 'pyin' : 'text',
131 131 'pyout' : 'text',
132 132 'pyerr' : 'text',
133 133 'stdout' : 'text',
134 134 'stderr' : 'text',
135 135 })
136 136
137 137 def __init__(self, **kwargs):
138 138 super(SQLiteDB, self).__init__(**kwargs)
139 139 if not self.table:
140 140 # use session, and prefix _, since starting with # is illegal
141 141 self.table = '_'+self.session.replace('-','_')
142 142 if not self.location:
143 143 # get current profile
144 from IPython.core.newapplication import BaseIPythonApplication
144 from IPython.core.application import BaseIPythonApplication
145 145 if BaseIPythonApplication.initialized():
146 146 app = BaseIPythonApplication.instance()
147 147 if app.profile_dir is not None:
148 148 self.location = app.profile_dir.location
149 149 else:
150 150 self.location = u'.'
151 151 else:
152 152 self.location = u'.'
153 153 self._init_db()
154 154
155 155 # register db commit as 2s periodic callback
156 156 # to prevent clogging pipes
157 157 # assumes we are being run in a zmq ioloop app
158 158 loop = ioloop.IOLoop.instance()
159 159 pc = ioloop.PeriodicCallback(self._db.commit, 2000, loop)
160 160 pc.start()
161 161
162 162 def _defaults(self, keys=None):
163 163 """create an empty record"""
164 164 d = {}
165 165 keys = self._keys if keys is None else keys
166 166 for key in keys:
167 167 d[key] = None
168 168 return d
169 169
170 170 def _check_table(self):
171 171 """Ensure that an incorrect table doesn't exist
172 172
173 173 If a bad (old) table does exist, return False
174 174 """
175 175 cursor = self._db.execute("PRAGMA table_info(%s)"%self.table)
176 176 lines = cursor.fetchall()
177 177 if not lines:
178 178 # table does not exist
179 179 return True
180 180 types = {}
181 181 keys = []
182 182 for line in lines:
183 183 keys.append(line[1])
184 184 types[line[1]] = line[2]
185 185 if self._keys != keys:
186 186 # key mismatch
187 187 self.log.warn('keys mismatch')
188 188 return False
189 189 for key in self._keys:
190 190 if types[key] != self._types[key]:
191 191 self.log.warn(
192 192 'type mismatch: %s: %s != %s'%(key,types[key],self._types[key])
193 193 )
194 194 return False
195 195 return True
196 196
197 197 def _init_db(self):
198 198 """Connect to the database and get new session number."""
199 199 # register adapters
200 200 sqlite3.register_adapter(dict, _adapt_dict)
201 201 sqlite3.register_converter('dict', _convert_dict)
202 202 sqlite3.register_adapter(list, _adapt_bufs)
203 203 sqlite3.register_converter('bufs', _convert_bufs)
204 204 # connect to the db
205 205 dbfile = os.path.join(self.location, self.filename)
206 206 self._db = sqlite3.connect(dbfile, detect_types=sqlite3.PARSE_DECLTYPES,
207 207 # isolation_level = None)#,
208 208 cached_statements=64)
209 209 # print dir(self._db)
210 210 first_table = self.table
211 211 i=0
212 212 while not self._check_table():
213 213 i+=1
214 214 self.table = first_table+'_%i'%i
215 215 self.log.warn(
216 216 "Table %s exists and doesn't match db format, trying %s"%
217 217 (first_table,self.table)
218 218 )
219 219
220 220 self._db.execute("""CREATE TABLE IF NOT EXISTS %s
221 221 (msg_id text PRIMARY KEY,
222 222 header dict text,
223 223 content dict text,
224 224 buffers bufs blob,
225 225 submitted timestamp,
226 226 client_uuid text,
227 227 engine_uuid text,
228 228 started timestamp,
229 229 completed timestamp,
230 230 resubmitted timestamp,
231 231 result_header dict text,
232 232 result_content dict text,
233 233 result_buffers bufs blob,
234 234 queue text,
235 235 pyin text,
236 236 pyout text,
237 237 pyerr text,
238 238 stdout text,
239 239 stderr text)
240 240 """%self.table)
241 241 self._db.commit()
242 242
243 243 def _dict_to_list(self, d):
244 244 """turn a mongodb-style record dict into a list."""
245 245
246 246 return [ d[key] for key in self._keys ]
247 247
248 248 def _list_to_dict(self, line, keys=None):
249 249 """Inverse of dict_to_list"""
250 250 keys = self._keys if keys is None else keys
251 251 d = self._defaults(keys)
252 252 for key,value in zip(keys, line):
253 253 d[key] = value
254 254
255 255 return d
256 256
257 257 def _render_expression(self, check):
258 258 """Turn a mongodb-style search dict into an SQL query."""
259 259 expressions = []
260 260 args = []
261 261
262 262 skeys = set(check.keys())
263 263 skeys.difference_update(set(self._keys))
264 264 skeys.difference_update(set(['buffers', 'result_buffers']))
265 265 if skeys:
266 266 raise KeyError("Illegal testing key(s): %s"%skeys)
267 267
268 268 for name,sub_check in check.iteritems():
269 269 if isinstance(sub_check, dict):
270 270 for test,value in sub_check.iteritems():
271 271 try:
272 272 op = operators[test]
273 273 except KeyError:
274 274 raise KeyError("Unsupported operator: %r"%test)
275 275 if isinstance(op, tuple):
276 276 op, join = op
277 277
278 278 if value is None and op in null_operators:
279 279 expr = "%s %s"%null_operators[op]
280 280 else:
281 281 expr = "%s %s ?"%(name, op)
282 282 if isinstance(value, (tuple,list)):
283 283 if op in null_operators and any([v is None for v in value]):
284 284 # equality tests don't work with NULL
285 285 raise ValueError("Cannot use %r test with NULL values on SQLite backend"%test)
286 286 expr = '( %s )'%( join.join([expr]*len(value)) )
287 287 args.extend(value)
288 288 else:
289 289 args.append(value)
290 290 expressions.append(expr)
291 291 else:
292 292 # it's an equality check
293 293 if sub_check is None:
294 294 expressions.append("%s IS NULL")
295 295 else:
296 296 expressions.append("%s = ?"%name)
297 297 args.append(sub_check)
298 298
299 299 expr = " AND ".join(expressions)
300 300 return expr, args
301 301
302 302 def add_record(self, msg_id, rec):
303 303 """Add a new Task Record, by msg_id."""
304 304 d = self._defaults()
305 305 d.update(rec)
306 306 d['msg_id'] = msg_id
307 307 line = self._dict_to_list(d)
308 308 tups = '(%s)'%(','.join(['?']*len(line)))
309 309 self._db.execute("INSERT INTO %s VALUES %s"%(self.table, tups), line)
310 310 # self._db.commit()
311 311
312 312 def get_record(self, msg_id):
313 313 """Get a specific Task Record, by msg_id."""
314 314 cursor = self._db.execute("""SELECT * FROM %s WHERE msg_id==?"""%self.table, (msg_id,))
315 315 line = cursor.fetchone()
316 316 if line is None:
317 317 raise KeyError("No such msg: %r"%msg_id)
318 318 return self._list_to_dict(line)
319 319
320 320 def update_record(self, msg_id, rec):
321 321 """Update the data in an existing record."""
322 322 query = "UPDATE %s SET "%self.table
323 323 sets = []
324 324 keys = sorted(rec.keys())
325 325 values = []
326 326 for key in keys:
327 327 sets.append('%s = ?'%key)
328 328 values.append(rec[key])
329 329 query += ', '.join(sets)
330 330 query += ' WHERE msg_id == ?'
331 331 values.append(msg_id)
332 332 self._db.execute(query, values)
333 333 # self._db.commit()
334 334
335 335 def drop_record(self, msg_id):
336 336 """Remove a record from the DB."""
337 337 self._db.execute("""DELETE FROM %s WHERE msg_id==?"""%self.table, (msg_id,))
338 338 # self._db.commit()
339 339
340 340 def drop_matching_records(self, check):
341 341 """Remove a record from the DB."""
342 342 expr,args = self._render_expression(check)
343 343 query = "DELETE FROM %s WHERE %s"%(self.table, expr)
344 344 self._db.execute(query,args)
345 345 # self._db.commit()
346 346
347 347 def find_records(self, check, keys=None):
348 348 """Find records matching a query dict, optionally extracting subset of keys.
349 349
350 350 Returns list of matching records.
351 351
352 352 Parameters
353 353 ----------
354 354
355 355 check: dict
356 356 mongodb-style query argument
357 357 keys: list of strs [optional]
358 358 if specified, the subset of keys to extract. msg_id will *always* be
359 359 included.
360 360 """
361 361 if keys:
362 362 bad_keys = [ key for key in keys if key not in self._keys ]
363 363 if bad_keys:
364 364 raise KeyError("Bad record key(s): %s"%bad_keys)
365 365
366 366 if keys:
367 367 # ensure msg_id is present and first:
368 368 if 'msg_id' in keys:
369 369 keys.remove('msg_id')
370 370 keys.insert(0, 'msg_id')
371 371 req = ', '.join(keys)
372 372 else:
373 373 req = '*'
374 374 expr,args = self._render_expression(check)
375 375 query = """SELECT %s FROM %s WHERE %s"""%(req, self.table, expr)
376 376 cursor = self._db.execute(query, args)
377 377 matches = cursor.fetchall()
378 378 records = []
379 379 for line in matches:
380 380 rec = self._list_to_dict(line, keys)
381 381 records.append(rec)
382 382 return records
383 383
384 384 def get_history(self):
385 385 """get all msg_ids, ordered by time submitted."""
386 386 query = """SELECT msg_id FROM %s ORDER by submitted ASC"""%self.table
387 387 cursor = self._db.execute(query)
388 388 # will be a list of length 1 tuples
389 389 return [ tup[0] for tup in cursor.fetchall()]
390 390
391 391 __all__ = ['SQLiteDB'] No newline at end of file
@@ -1,673 +1,673 b''
1 1 #!/usr/bin/env python
2 2 """A simple interactive kernel that talks to a frontend over 0MQ.
3 3
4 4 Things to do:
5 5
6 6 * Implement `set_parent` logic. Right before doing exec, the Kernel should
7 7 call set_parent on all the PUB objects with the message about to be executed.
8 8 * Implement random port and security key logic.
9 9 * Implement control messages.
10 10 * Implement event loop and poll version.
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Imports
15 15 #-----------------------------------------------------------------------------
16 16 from __future__ import print_function
17 17
18 18 # Standard library imports.
19 19 import __builtin__
20 20 import atexit
21 21 import sys
22 22 import time
23 23 import traceback
24 24 import logging
25 25 # System library imports.
26 26 import zmq
27 27
28 28 # Local imports.
29 29 from IPython.config.configurable import Configurable
30 30 from IPython.config.application import boolean_flag
31 from IPython.core.newapplication import ProfileDir
31 from IPython.core.application import ProfileDir
32 32 from IPython.core.shellapp import (
33 33 InteractiveShellApp, shell_flags, shell_aliases
34 34 )
35 35 from IPython.utils import io
36 36 from IPython.utils.jsonutil import json_clean
37 37 from IPython.lib import pylabtools
38 38 from IPython.utils.traitlets import (
39 39 List, Instance, Float, Dict, Bool, Int, Unicode, CaselessStrEnum
40 40 )
41 41
42 42 from entry_point import base_launch_kernel
43 43 from kernelapp import KernelApp, kernel_flags, kernel_aliases
44 44 from iostream import OutStream
45 45 from session import Session, Message
46 46 from zmqshell import ZMQInteractiveShell
47 47
48 48
49 49
50 50 #-----------------------------------------------------------------------------
51 51 # Main kernel class
52 52 #-----------------------------------------------------------------------------
53 53
54 54 class Kernel(Configurable):
55 55
56 56 #---------------------------------------------------------------------------
57 57 # Kernel interface
58 58 #---------------------------------------------------------------------------
59 59
60 60 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
61 61 session = Instance(Session)
62 62 shell_socket = Instance('zmq.Socket')
63 63 iopub_socket = Instance('zmq.Socket')
64 64 stdin_socket = Instance('zmq.Socket')
65 65 log = Instance(logging.Logger)
66 66
67 67 # Private interface
68 68
69 69 # Time to sleep after flushing the stdout/err buffers in each execute
70 70 # cycle. While this introduces a hard limit on the minimal latency of the
71 71 # execute cycle, it helps prevent output synchronization problems for
72 72 # clients.
73 73 # Units are in seconds. The minimum zmq latency on local host is probably
74 74 # ~150 microseconds, set this to 500us for now. We may need to increase it
75 75 # a little if it's not enough after more interactive testing.
76 76 _execute_sleep = Float(0.0005, config=True)
77 77
78 78 # Frequency of the kernel's event loop.
79 79 # Units are in seconds, kernel subclasses for GUI toolkits may need to
80 80 # adapt to milliseconds.
81 81 _poll_interval = Float(0.05, config=True)
82 82
83 83 # If the shutdown was requested over the network, we leave here the
84 84 # necessary reply message so it can be sent by our registered atexit
85 85 # handler. This ensures that the reply is only sent to clients truly at
86 86 # the end of our shutdown process (which happens after the underlying
87 87 # IPython shell's own shutdown).
88 88 _shutdown_message = None
89 89
90 90 # This is a dict of port number that the kernel is listening on. It is set
91 91 # by record_ports and used by connect_request.
92 92 _recorded_ports = Dict()
93 93
94 94
95 95
96 96 def __init__(self, **kwargs):
97 97 super(Kernel, self).__init__(**kwargs)
98 98
99 99 # Before we even start up the shell, register *first* our exit handlers
100 100 # so they come before the shell's
101 101 atexit.register(self._at_shutdown)
102 102
103 103 # Initialize the InteractiveShell subclass
104 104 self.shell = ZMQInteractiveShell.instance(config=self.config)
105 105 self.shell.displayhook.session = self.session
106 106 self.shell.displayhook.pub_socket = self.iopub_socket
107 107 self.shell.display_pub.session = self.session
108 108 self.shell.display_pub.pub_socket = self.iopub_socket
109 109
110 110 # TMP - hack while developing
111 111 self.shell._reply_content = None
112 112
113 113 # Build dict of handlers for message types
114 114 msg_types = [ 'execute_request', 'complete_request',
115 115 'object_info_request', 'history_request',
116 116 'connect_request', 'shutdown_request']
117 117 self.handlers = {}
118 118 for msg_type in msg_types:
119 119 self.handlers[msg_type] = getattr(self, msg_type)
120 120
121 121 def do_one_iteration(self):
122 122 """Do one iteration of the kernel's evaluation loop.
123 123 """
124 124 ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
125 125 if msg is None:
126 126 return
127 127
128 128 # This assert will raise in versions of zeromq 2.0.7 and lesser.
129 129 # We now require 2.0.8 or above, so we can uncomment for safety.
130 130 # print(ident,msg, file=sys.__stdout__)
131 131 assert ident is not None, "Missing message part."
132 132
133 133 # Print some info about this message and leave a '--->' marker, so it's
134 134 # easier to trace visually the message chain when debugging. Each
135 135 # handler prints its message at the end.
136 136 self.log.debug('\n*** MESSAGE TYPE:'+str(msg['msg_type'])+'***')
137 137 self.log.debug(' Content: '+str(msg['content'])+'\n --->\n ')
138 138
139 139 # Find and call actual handler for message
140 140 handler = self.handlers.get(msg['msg_type'], None)
141 141 if handler is None:
142 142 self.log.error("UNKNOWN MESSAGE TYPE:" +str(msg))
143 143 else:
144 144 handler(ident, msg)
145 145
146 146 # Check whether we should exit, in case the incoming message set the
147 147 # exit flag on
148 148 if self.shell.exit_now:
149 149 self.log.debug('\nExiting IPython kernel...')
150 150 # We do a normal, clean exit, which allows any actions registered
151 151 # via atexit (such as history saving) to take place.
152 152 sys.exit(0)
153 153
154 154
155 155 def start(self):
156 156 """ Start the kernel main loop.
157 157 """
158 158 poller = zmq.Poller()
159 159 poller.register(self.shell_socket, zmq.POLLIN)
160 160 while True:
161 161 try:
162 162 # scale by extra factor of 10, because there is no
163 163 # reason for this to be anything less than ~ 0.1s
164 164 # since it is a real poller and will respond
165 165 # to events immediately
166 166 poller.poll(10*1000*self._poll_interval)
167 167 self.do_one_iteration()
168 168 except KeyboardInterrupt:
169 169 # Ctrl-C shouldn't crash the kernel
170 170 io.raw_print("KeyboardInterrupt caught in kernel")
171 171
172 172 def record_ports(self, ports):
173 173 """Record the ports that this kernel is using.
174 174
175 175 The creator of the Kernel instance must call this methods if they
176 176 want the :meth:`connect_request` method to return the port numbers.
177 177 """
178 178 self._recorded_ports = ports
179 179
180 180 #---------------------------------------------------------------------------
181 181 # Kernel request handlers
182 182 #---------------------------------------------------------------------------
183 183
184 184 def _publish_pyin(self, code, parent):
185 185 """Publish the code request on the pyin stream."""
186 186
187 187 pyin_msg = self.session.send(self.iopub_socket, u'pyin',{u'code':code}, parent=parent)
188 188
189 189 def execute_request(self, ident, parent):
190 190
191 191 status_msg = self.session.send(self.iopub_socket,
192 192 u'status',
193 193 {u'execution_state':u'busy'},
194 194 parent=parent
195 195 )
196 196
197 197 try:
198 198 content = parent[u'content']
199 199 code = content[u'code']
200 200 silent = content[u'silent']
201 201 except:
202 202 self.log.error("Got bad msg: ")
203 203 self.log.error(str(Message(parent)))
204 204 return
205 205
206 206 shell = self.shell # we'll need this a lot here
207 207
208 208 # Replace raw_input. Note that is not sufficient to replace
209 209 # raw_input in the user namespace.
210 210 raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
211 211 __builtin__.raw_input = raw_input
212 212
213 213 # Set the parent message of the display hook and out streams.
214 214 shell.displayhook.set_parent(parent)
215 215 shell.display_pub.set_parent(parent)
216 216 sys.stdout.set_parent(parent)
217 217 sys.stderr.set_parent(parent)
218 218
219 219 # Re-broadcast our input for the benefit of listening clients, and
220 220 # start computing output
221 221 if not silent:
222 222 self._publish_pyin(code, parent)
223 223
224 224 reply_content = {}
225 225 try:
226 226 if silent:
227 227 # run_code uses 'exec' mode, so no displayhook will fire, and it
228 228 # doesn't call logging or history manipulations. Print
229 229 # statements in that code will obviously still execute.
230 230 shell.run_code(code)
231 231 else:
232 232 # FIXME: the shell calls the exception handler itself.
233 233 shell.run_cell(code)
234 234 except:
235 235 status = u'error'
236 236 # FIXME: this code right now isn't being used yet by default,
237 237 # because the run_cell() call above directly fires off exception
238 238 # reporting. This code, therefore, is only active in the scenario
239 239 # where runlines itself has an unhandled exception. We need to
240 240 # uniformize this, for all exception construction to come from a
241 241 # single location in the codbase.
242 242 etype, evalue, tb = sys.exc_info()
243 243 tb_list = traceback.format_exception(etype, evalue, tb)
244 244 reply_content.update(shell._showtraceback(etype, evalue, tb_list))
245 245 else:
246 246 status = u'ok'
247 247
248 248 reply_content[u'status'] = status
249 249
250 250 # Return the execution counter so clients can display prompts
251 251 reply_content['execution_count'] = shell.execution_count -1
252 252
253 253 # FIXME - fish exception info out of shell, possibly left there by
254 254 # runlines. We'll need to clean up this logic later.
255 255 if shell._reply_content is not None:
256 256 reply_content.update(shell._reply_content)
257 257 # reset after use
258 258 shell._reply_content = None
259 259
260 260 # At this point, we can tell whether the main code execution succeeded
261 261 # or not. If it did, we proceed to evaluate user_variables/expressions
262 262 if reply_content['status'] == 'ok':
263 263 reply_content[u'user_variables'] = \
264 264 shell.user_variables(content[u'user_variables'])
265 265 reply_content[u'user_expressions'] = \
266 266 shell.user_expressions(content[u'user_expressions'])
267 267 else:
268 268 # If there was an error, don't even try to compute variables or
269 269 # expressions
270 270 reply_content[u'user_variables'] = {}
271 271 reply_content[u'user_expressions'] = {}
272 272
273 273 # Payloads should be retrieved regardless of outcome, so we can both
274 274 # recover partial output (that could have been generated early in a
275 275 # block, before an error) and clear the payload system always.
276 276 reply_content[u'payload'] = shell.payload_manager.read_payload()
277 277 # Be agressive about clearing the payload because we don't want
278 278 # it to sit in memory until the next execute_request comes in.
279 279 shell.payload_manager.clear_payload()
280 280
281 281 # Flush output before sending the reply.
282 282 sys.stdout.flush()
283 283 sys.stderr.flush()
284 284 # FIXME: on rare occasions, the flush doesn't seem to make it to the
285 285 # clients... This seems to mitigate the problem, but we definitely need
286 286 # to better understand what's going on.
287 287 if self._execute_sleep:
288 288 time.sleep(self._execute_sleep)
289 289
290 290 # Send the reply.
291 291 reply_msg = self.session.send(self.shell_socket, u'execute_reply',
292 292 reply_content, parent, ident=ident)
293 293 self.log.debug(str(reply_msg))
294 294
295 295 if reply_msg['content']['status'] == u'error':
296 296 self._abort_queue()
297 297
298 298 status_msg = self.session.send(self.iopub_socket,
299 299 u'status',
300 300 {u'execution_state':u'idle'},
301 301 parent=parent
302 302 )
303 303
304 304 def complete_request(self, ident, parent):
305 305 txt, matches = self._complete(parent)
306 306 matches = {'matches' : matches,
307 307 'matched_text' : txt,
308 308 'status' : 'ok'}
309 309 completion_msg = self.session.send(self.shell_socket, 'complete_reply',
310 310 matches, parent, ident)
311 311 self.log.debug(str(completion_msg))
312 312
313 313 def object_info_request(self, ident, parent):
314 314 object_info = self.shell.object_inspect(parent['content']['oname'])
315 315 # Before we send this object over, we scrub it for JSON usage
316 316 oinfo = json_clean(object_info)
317 317 msg = self.session.send(self.shell_socket, 'object_info_reply',
318 318 oinfo, parent, ident)
319 319 self.log.debug(msg)
320 320
321 321 def history_request(self, ident, parent):
322 322 # We need to pull these out, as passing **kwargs doesn't work with
323 323 # unicode keys before Python 2.6.5.
324 324 hist_access_type = parent['content']['hist_access_type']
325 325 raw = parent['content']['raw']
326 326 output = parent['content']['output']
327 327 if hist_access_type == 'tail':
328 328 n = parent['content']['n']
329 329 hist = self.shell.history_manager.get_tail(n, raw=raw, output=output,
330 330 include_latest=True)
331 331
332 332 elif hist_access_type == 'range':
333 333 session = parent['content']['session']
334 334 start = parent['content']['start']
335 335 stop = parent['content']['stop']
336 336 hist = self.shell.history_manager.get_range(session, start, stop,
337 337 raw=raw, output=output)
338 338
339 339 elif hist_access_type == 'search':
340 340 pattern = parent['content']['pattern']
341 341 hist = self.shell.history_manager.search(pattern, raw=raw, output=output)
342 342
343 343 else:
344 344 hist = []
345 345 content = {'history' : list(hist)}
346 346 msg = self.session.send(self.shell_socket, 'history_reply',
347 347 content, parent, ident)
348 348 self.log.debug(str(msg))
349 349
350 350 def connect_request(self, ident, parent):
351 351 if self._recorded_ports is not None:
352 352 content = self._recorded_ports.copy()
353 353 else:
354 354 content = {}
355 355 msg = self.session.send(self.shell_socket, 'connect_reply',
356 356 content, parent, ident)
357 357 self.log.debug(msg)
358 358
359 359 def shutdown_request(self, ident, parent):
360 360 self.shell.exit_now = True
361 361 self._shutdown_message = self.session.msg(u'shutdown_reply', parent['content'], parent)
362 362 sys.exit(0)
363 363
364 364 #---------------------------------------------------------------------------
365 365 # Protected interface
366 366 #---------------------------------------------------------------------------
367 367
368 368 def _abort_queue(self):
369 369 while True:
370 370 ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
371 371 if msg is None:
372 372 break
373 373 else:
374 374 assert ident is not None, \
375 375 "Unexpected missing message part."
376 376
377 377 self.log.debug("Aborting:\n"+str(Message(msg)))
378 378 msg_type = msg['msg_type']
379 379 reply_type = msg_type.split('_')[0] + '_reply'
380 380 reply_msg = self.session.send(self.shell_socket, reply_type,
381 381 {'status' : 'aborted'}, msg, ident=ident)
382 382 self.log.debug(reply_msg)
383 383 # We need to wait a bit for requests to come in. This can probably
384 384 # be set shorter for true asynchronous clients.
385 385 time.sleep(0.1)
386 386
387 387 def _raw_input(self, prompt, ident, parent):
388 388 # Flush output before making the request.
389 389 sys.stderr.flush()
390 390 sys.stdout.flush()
391 391
392 392 # Send the input request.
393 393 content = dict(prompt=prompt)
394 394 msg = self.session.send(self.stdin_socket, u'input_request', content, parent)
395 395
396 396 # Await a response.
397 397 ident, reply = self.session.recv(self.stdin_socket, 0)
398 398 try:
399 399 value = reply['content']['value']
400 400 except:
401 401 self.log.error("Got bad raw_input reply: ")
402 402 self.log.error(str(Message(parent)))
403 403 value = ''
404 404 return value
405 405
406 406 def _complete(self, msg):
407 407 c = msg['content']
408 408 try:
409 409 cpos = int(c['cursor_pos'])
410 410 except:
411 411 # If we don't get something that we can convert to an integer, at
412 412 # least attempt the completion guessing the cursor is at the end of
413 413 # the text, if there's any, and otherwise of the line
414 414 cpos = len(c['text'])
415 415 if cpos==0:
416 416 cpos = len(c['line'])
417 417 return self.shell.complete(c['text'], c['line'], cpos)
418 418
419 419 def _object_info(self, context):
420 420 symbol, leftover = self._symbol_from_context(context)
421 421 if symbol is not None and not leftover:
422 422 doc = getattr(symbol, '__doc__', '')
423 423 else:
424 424 doc = ''
425 425 object_info = dict(docstring = doc)
426 426 return object_info
427 427
428 428 def _symbol_from_context(self, context):
429 429 if not context:
430 430 return None, context
431 431
432 432 base_symbol_string = context[0]
433 433 symbol = self.shell.user_ns.get(base_symbol_string, None)
434 434 if symbol is None:
435 435 symbol = __builtin__.__dict__.get(base_symbol_string, None)
436 436 if symbol is None:
437 437 return None, context
438 438
439 439 context = context[1:]
440 440 for i, name in enumerate(context):
441 441 new_symbol = getattr(symbol, name, None)
442 442 if new_symbol is None:
443 443 return symbol, context[i:]
444 444 else:
445 445 symbol = new_symbol
446 446
447 447 return symbol, []
448 448
449 449 def _at_shutdown(self):
450 450 """Actions taken at shutdown by the kernel, called by python's atexit.
451 451 """
452 452 # io.rprint("Kernel at_shutdown") # dbg
453 453 if self._shutdown_message is not None:
454 454 self.session.send(self.shell_socket, self._shutdown_message)
455 455 self.session.send(self.iopub_socket, self._shutdown_message)
456 456 self.log.debug(str(self._shutdown_message))
457 457 # A very short sleep to give zmq time to flush its message buffers
458 458 # before Python truly shuts down.
459 459 time.sleep(0.01)
460 460
461 461
462 462 class QtKernel(Kernel):
463 463 """A Kernel subclass with Qt support."""
464 464
465 465 def start(self):
466 466 """Start a kernel with QtPy4 event loop integration."""
467 467
468 468 from PyQt4 import QtCore
469 469 from IPython.lib.guisupport import get_app_qt4, start_event_loop_qt4
470 470
471 471 self.app = get_app_qt4([" "])
472 472 self.app.setQuitOnLastWindowClosed(False)
473 473 self.timer = QtCore.QTimer()
474 474 self.timer.timeout.connect(self.do_one_iteration)
475 475 # Units for the timer are in milliseconds
476 476 self.timer.start(1000*self._poll_interval)
477 477 start_event_loop_qt4(self.app)
478 478
479 479
480 480 class WxKernel(Kernel):
481 481 """A Kernel subclass with Wx support."""
482 482
483 483 def start(self):
484 484 """Start a kernel with wx event loop support."""
485 485
486 486 import wx
487 487 from IPython.lib.guisupport import start_event_loop_wx
488 488
489 489 doi = self.do_one_iteration
490 490 # Wx uses milliseconds
491 491 poll_interval = int(1000*self._poll_interval)
492 492
493 493 # We have to put the wx.Timer in a wx.Frame for it to fire properly.
494 494 # We make the Frame hidden when we create it in the main app below.
495 495 class TimerFrame(wx.Frame):
496 496 def __init__(self, func):
497 497 wx.Frame.__init__(self, None, -1)
498 498 self.timer = wx.Timer(self)
499 499 # Units for the timer are in milliseconds
500 500 self.timer.Start(poll_interval)
501 501 self.Bind(wx.EVT_TIMER, self.on_timer)
502 502 self.func = func
503 503
504 504 def on_timer(self, event):
505 505 self.func()
506 506
507 507 # We need a custom wx.App to create our Frame subclass that has the
508 508 # wx.Timer to drive the ZMQ event loop.
509 509 class IPWxApp(wx.App):
510 510 def OnInit(self):
511 511 self.frame = TimerFrame(doi)
512 512 self.frame.Show(False)
513 513 return True
514 514
515 515 # The redirect=False here makes sure that wx doesn't replace
516 516 # sys.stdout/stderr with its own classes.
517 517 self.app = IPWxApp(redirect=False)
518 518 start_event_loop_wx(self.app)
519 519
520 520
521 521 class TkKernel(Kernel):
522 522 """A Kernel subclass with Tk support."""
523 523
524 524 def start(self):
525 525 """Start a Tk enabled event loop."""
526 526
527 527 import Tkinter
528 528 doi = self.do_one_iteration
529 529 # Tk uses milliseconds
530 530 poll_interval = int(1000*self._poll_interval)
531 531 # For Tkinter, we create a Tk object and call its withdraw method.
532 532 class Timer(object):
533 533 def __init__(self, func):
534 534 self.app = Tkinter.Tk()
535 535 self.app.withdraw()
536 536 self.func = func
537 537
538 538 def on_timer(self):
539 539 self.func()
540 540 self.app.after(poll_interval, self.on_timer)
541 541
542 542 def start(self):
543 543 self.on_timer() # Call it once to get things going.
544 544 self.app.mainloop()
545 545
546 546 self.timer = Timer(doi)
547 547 self.timer.start()
548 548
549 549
550 550 class GTKKernel(Kernel):
551 551 """A Kernel subclass with GTK support."""
552 552
553 553 def start(self):
554 554 """Start the kernel, coordinating with the GTK event loop"""
555 555 from .gui.gtkembed import GTKEmbed
556 556
557 557 gtk_kernel = GTKEmbed(self)
558 558 gtk_kernel.start()
559 559
560 560
561 561 #-----------------------------------------------------------------------------
562 562 # Aliases and Flags for the IPKernelApp
563 563 #-----------------------------------------------------------------------------
564 564
565 565 flags = dict(kernel_flags)
566 566 flags.update(shell_flags)
567 567
568 568 addflag = lambda *args: flags.update(boolean_flag(*args))
569 569
570 570 flags['pylab'] = (
571 571 {'IPKernelApp' : {'pylab' : 'auto'}},
572 572 """Pre-load matplotlib and numpy for interactive use with
573 573 the default matplotlib backend."""
574 574 )
575 575
576 576 aliases = dict(kernel_aliases)
577 577 aliases.update(shell_aliases)
578 578
579 579 # it's possible we don't want short aliases for *all* of these:
580 580 aliases.update(dict(
581 581 pylab='IPKernelApp.pylab',
582 582 ))
583 583
584 584 #-----------------------------------------------------------------------------
585 585 # The IPKernelApp class
586 586 #-----------------------------------------------------------------------------
587 587
588 588 class IPKernelApp(KernelApp, InteractiveShellApp):
589 589 name = 'ipkernel'
590 590
591 591 aliases = Dict(aliases)
592 592 flags = Dict(flags)
593 593 classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session]
594 594 # configurables
595 595 pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'],
596 596 config=True,
597 597 help="""Pre-load matplotlib and numpy for interactive use,
598 598 selecting a particular matplotlib backend and loop integration.
599 599 """
600 600 )
601 601 def initialize(self, argv=None):
602 602 super(IPKernelApp, self).initialize(argv)
603 603 self.init_shell()
604 604 self.init_extensions()
605 605 self.init_code()
606 606
607 607 def init_kernel(self):
608 608 kernel_factory = Kernel
609 609
610 610 kernel_map = {
611 611 'qt' : QtKernel,
612 612 'qt4': QtKernel,
613 613 'inline': Kernel,
614 614 'osx': TkKernel,
615 615 'wx' : WxKernel,
616 616 'tk' : TkKernel,
617 617 'gtk': GTKKernel,
618 618 }
619 619
620 620 if self.pylab:
621 621 key = None if self.pylab == 'auto' else self.pylab
622 622 gui, backend = pylabtools.find_gui_and_backend(key)
623 623 kernel_factory = kernel_map.get(gui)
624 624 if kernel_factory is None:
625 625 raise ValueError('GUI is not supported: %r' % gui)
626 626 pylabtools.activate_matplotlib(backend)
627 627
628 628 kernel = kernel_factory(config=self.config, session=self.session,
629 629 shell_socket=self.shell_socket,
630 630 iopub_socket=self.iopub_socket,
631 631 stdin_socket=self.stdin_socket,
632 632 log=self.log
633 633 )
634 634 self.kernel = kernel
635 635 kernel.record_ports(self.ports)
636 636
637 637 if self.pylab:
638 638 pylabtools.import_pylab(kernel.shell.user_ns, backend,
639 639 shell=kernel.shell)
640 640
641 641 def init_shell(self):
642 642 self.shell = self.kernel.shell
643 643
644 644
645 645 #-----------------------------------------------------------------------------
646 646 # Kernel main and launch functions
647 647 #-----------------------------------------------------------------------------
648 648
649 649 def launch_kernel(*args, **kwargs):
650 650 """Launches a localhost IPython kernel, binding to the specified ports.
651 651
652 652 This function simply calls entry_point.base_launch_kernel with the right first
653 653 command to start an ipkernel. See base_launch_kernel for arguments.
654 654
655 655 Returns
656 656 -------
657 657 A tuple of form:
658 658 (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
659 659 where kernel_process is a Popen object and the ports are integers.
660 660 """
661 661 return base_launch_kernel('from IPython.zmq.ipkernel import main; main()',
662 662 *args, **kwargs)
663 663
664 664
665 665 def main():
666 666 """Run an IPKernel as an application"""
667 667 app = IPKernelApp.instance()
668 668 app.initialize()
669 669 app.start()
670 670
671 671
672 672 if __name__ == '__main__':
673 673 main()
@@ -1,215 +1,215 b''
1 1 #!/usr/bin/env python
2 2 """An Application for launching a kernel
3 3
4 4 Authors
5 5 -------
6 6 * MinRK
7 7 """
8 8 #-----------------------------------------------------------------------------
9 9 # Copyright (C) 2011 The IPython Development Team
10 10 #
11 11 # Distributed under the terms of the BSD License. The full license is in
12 12 # the file COPYING.txt, distributed as part of this software.
13 13 #-----------------------------------------------------------------------------
14 14
15 15 #-----------------------------------------------------------------------------
16 16 # Imports
17 17 #-----------------------------------------------------------------------------
18 18
19 19 # Standard library imports.
20 20 import os
21 21 import sys
22 22
23 23 # System library imports.
24 24 import zmq
25 25
26 26 # IPython imports.
27 27 from IPython.core.ultratb import FormattedTB
28 from IPython.core.newapplication import (
28 from IPython.core.application import (
29 29 BaseIPythonApplication, base_flags, base_aliases
30 30 )
31 31 from IPython.utils import io
32 32 from IPython.utils.localinterfaces import LOCALHOST
33 33 from IPython.utils.traitlets import Any, Instance, Dict, Unicode, Int, Bool
34 34 from IPython.utils.importstring import import_item
35 35 # local imports
36 36 from IPython.zmq.heartbeat import Heartbeat
37 37 from IPython.zmq.parentpoller import ParentPollerUnix, ParentPollerWindows
38 38 from IPython.zmq.session import Session
39 39
40 40
41 41 #-----------------------------------------------------------------------------
42 42 # Flags and Aliases
43 43 #-----------------------------------------------------------------------------
44 44
45 45 kernel_aliases = dict(base_aliases)
46 46 kernel_aliases.update({
47 47 'ip' : 'KernelApp.ip',
48 48 'hb' : 'KernelApp.hb_port',
49 49 'shell' : 'KernelApp.shell_port',
50 50 'iopub' : 'KernelApp.iopub_port',
51 51 'stdin' : 'KernelApp.stdin_port',
52 52 'parent': 'KernelApp.parent',
53 53 })
54 54 if sys.platform.startswith('win'):
55 55 kernel_aliases['interrupt'] = 'KernelApp.interrupt'
56 56
57 57 kernel_flags = dict(base_flags)
58 58 kernel_flags.update({
59 59 'no-stdout' : (
60 60 {'KernelApp' : {'no_stdout' : True}},
61 61 "redirect stdout to the null device"),
62 62 'no-stderr' : (
63 63 {'KernelApp' : {'no_stderr' : True}},
64 64 "redirect stderr to the null device"),
65 65 })
66 66
67 67
68 68 #-----------------------------------------------------------------------------
69 69 # Application class for starting a Kernel
70 70 #-----------------------------------------------------------------------------
71 71
72 72 class KernelApp(BaseIPythonApplication):
73 73 name='pykernel'
74 74 aliases = Dict(kernel_aliases)
75 75 flags = Dict(kernel_flags)
76 76 classes = [Session]
77 77 # the kernel class, as an importstring
78 78 kernel_class = Unicode('IPython.zmq.pykernel.Kernel')
79 79 kernel = Any()
80 80 poller = Any() # don't restrict this even though current pollers are all Threads
81 81 heartbeat = Instance(Heartbeat)
82 82 session = Instance('IPython.zmq.session.Session')
83 83 ports = Dict()
84 84
85 85 # connection info:
86 86 ip = Unicode(LOCALHOST, config=True,
87 87 help="Set the IP or interface on which the kernel will listen.")
88 88 hb_port = Int(0, config=True, help="set the heartbeat port [default: random]")
89 89 shell_port = Int(0, config=True, help="set the shell (XREP) port [default: random]")
90 90 iopub_port = Int(0, config=True, help="set the iopub (PUB) port [default: random]")
91 91 stdin_port = Int(0, config=True, help="set the stdin (XREQ) port [default: random]")
92 92
93 93 # streams, etc.
94 94 no_stdout = Bool(False, config=True, help="redirect stdout to the null device")
95 95 no_stderr = Bool(False, config=True, help="redirect stderr to the null device")
96 96 outstream_class = Unicode('IPython.zmq.iostream.OutStream', config=True,
97 97 help="The importstring for the OutStream factory")
98 98 displayhook_class = Unicode('IPython.zmq.displayhook.DisplayHook', config=True,
99 99 help="The importstring for the DisplayHook factory")
100 100
101 101 # polling
102 102 parent = Int(0, config=True,
103 103 help="""kill this process if its parent dies. On Windows, the argument
104 104 specifies the HANDLE of the parent process, otherwise it is simply boolean.
105 105 """)
106 106 interrupt = Int(0, config=True,
107 107 help="""ONLY USED ON WINDOWS
108 108 Interrupt this process when the parent is signalled.
109 109 """)
110 110
111 111 def init_crash_handler(self):
112 112 # Install minimal exception handling
113 113 sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor',
114 114 ostream=sys.__stdout__)
115 115
116 116 def init_poller(self):
117 117 if sys.platform == 'win32':
118 118 if self.interrupt or self.parent:
119 119 self.poller = ParentPollerWindows(self.interrupt, self.parent)
120 120 elif self.parent:
121 121 self.poller = ParentPollerUnix()
122 122
123 123 def _bind_socket(self, s, port):
124 124 iface = 'tcp://%s' % self.ip
125 125 if port <= 0:
126 126 port = s.bind_to_random_port(iface)
127 127 else:
128 128 s.bind(iface + ':%i'%port)
129 129 return port
130 130
131 131 def init_sockets(self):
132 132 # Create a context, a session, and the kernel sockets.
133 133 io.raw_print("Starting the kernel at pid:", os.getpid())
134 134 context = zmq.Context.instance()
135 135 # Uncomment this to try closing the context.
136 136 # atexit.register(context.term)
137 137
138 138 self.shell_socket = context.socket(zmq.XREP)
139 139 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port)
140 140 self.log.debug("shell XREP Channel on port: %i"%self.shell_port)
141 141
142 142 self.iopub_socket = context.socket(zmq.PUB)
143 143 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port)
144 144 self.log.debug("iopub PUB Channel on port: %i"%self.iopub_port)
145 145
146 146 self.stdin_socket = context.socket(zmq.XREQ)
147 147 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
148 148 self.log.debug("stdin XREQ Channel on port: %i"%self.stdin_port)
149 149
150 150 self.heartbeat = Heartbeat(context, (self.ip, self.hb_port))
151 151 self.hb_port = self.heartbeat.port
152 152 self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port)
153 153
154 154 # Helper to make it easier to connect to an existing kernel, until we have
155 155 # single-port connection negotiation fully implemented.
156 156 self.log.info("To connect another client to this kernel, use:")
157 157 self.log.info("--external shell={0} iopub={1} stdin={2} hb={3}".format(
158 158 self.shell_port, self.iopub_port, self.stdin_port, self.hb_port))
159 159
160 160
161 161 self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
162 162 stdin=self.stdin_port, hb=self.hb_port)
163 163
164 164 def init_session(self):
165 165 """create our session object"""
166 166 self.session = Session(config=self.config, username=u'kernel')
167 167
168 168 def init_io(self):
169 169 """redirects stdout/stderr, and installs a display hook"""
170 170 # Re-direct stdout/stderr, if necessary.
171 171 if self.no_stdout or self.no_stderr:
172 172 blackhole = file(os.devnull, 'w')
173 173 if self.no_stdout:
174 174 sys.stdout = sys.__stdout__ = blackhole
175 175 if self.no_stderr:
176 176 sys.stderr = sys.__stderr__ = blackhole
177 177
178 178 # Redirect input streams and set a display hook.
179 179
180 180 if self.outstream_class:
181 181 outstream_factory = import_item(str(self.outstream_class))
182 182 sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout')
183 183 sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr')
184 184 if self.displayhook_class:
185 185 displayhook_factory = import_item(str(self.displayhook_class))
186 186 sys.displayhook = displayhook_factory(self.session, self.iopub_socket)
187 187
188 188 def init_kernel(self):
189 189 """Create the Kernel object itself"""
190 190 kernel_factory = import_item(str(self.kernel_class))
191 191 self.kernel = kernel_factory(config=self.config, session=self.session,
192 192 shell_socket=self.shell_socket,
193 193 iopub_socket=self.iopub_socket,
194 194 stdin_socket=self.stdin_socket,
195 195 log=self.log
196 196 )
197 197 self.kernel.record_ports(self.ports)
198 198
199 199 def initialize(self, argv=None):
200 200 super(KernelApp, self).initialize(argv)
201 201 self.init_session()
202 202 self.init_poller()
203 203 self.init_sockets()
204 204 self.init_io()
205 205 self.init_kernel()
206 206
207 207 def start(self):
208 208 self.heartbeat.start()
209 209 if self.poller is not None:
210 210 self.poller.start()
211 211 try:
212 212 self.kernel.start()
213 213 except KeyboardInterrupt:
214 214 pass
215 215
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now