##// END OF EJS Templates
rename py3compat.getcwd() -> os.getcwd()
Srinivas Reddy Thatiparthy -
Show More
@@ -1,458 +1,458 b''
1 1 # encoding: utf-8
2 2 """
3 3 An application for IPython.
4 4
5 5 All top-level applications should use the classes in this module for
6 6 handling configuration and creating configurables.
7 7
8 8 The job of an :class:`Application` is to create the master configuration
9 9 object and then create the configurable objects, passing the config to them.
10 10 """
11 11
12 12 # Copyright (c) IPython Development Team.
13 13 # Distributed under the terms of the Modified BSD License.
14 14
15 15 import atexit
16 16 from copy import deepcopy
17 17 import glob
18 18 import logging
19 19 import os
20 20 import shutil
21 21 import sys
22 22
23 23 from traitlets.config.application import Application, catch_config_error
24 24 from traitlets.config.loader import ConfigFileNotFound, PyFileConfigLoader
25 25 from IPython.core import release, crashhandler
26 26 from IPython.core.profiledir import ProfileDir, ProfileDirError
27 27 from IPython.paths import get_ipython_dir, get_ipython_package_dir
28 28 from IPython.utils.path import ensure_dir_exists
29 29 from IPython.utils import py3compat
30 30 from traitlets import (
31 31 List, Unicode, Type, Bool, Dict, Set, Instance, Undefined,
32 32 default, observe,
33 33 )
34 34
35 35 if os.name == 'nt':
36 36 programdata = os.environ.get('PROGRAMDATA', None)
37 37 if programdata:
38 38 SYSTEM_CONFIG_DIRS = [os.path.join(programdata, 'ipython')]
39 39 else: # PROGRAMDATA is not defined by default on XP.
40 40 SYSTEM_CONFIG_DIRS = []
41 41 else:
42 42 SYSTEM_CONFIG_DIRS = [
43 43 "/usr/local/etc/ipython",
44 44 "/etc/ipython",
45 45 ]
46 46
47 47 _envvar = os.environ.get('IPYTHON_SUPPRESS_CONFIG_ERRORS')
48 48 if _envvar in {None, ''}:
49 49 IPYTHON_SUPPRESS_CONFIG_ERRORS = None
50 50 else:
51 51 if _envvar.lower() in {'1','true'}:
52 52 IPYTHON_SUPPRESS_CONFIG_ERRORS = True
53 53 elif _envvar.lower() in {'0','false'} :
54 54 IPYTHON_SUPPRESS_CONFIG_ERRORS = False
55 55 else:
56 56 sys.exit("Unsupported value for environment variable: 'IPYTHON_SUPPRESS_CONFIG_ERRORS' is set to '%s' which is none of {'0', '1', 'false', 'true', ''}."% _envvar )
57 57
58 58 # aliases and flags
59 59
60 60 base_aliases = {
61 61 'profile-dir' : 'ProfileDir.location',
62 62 'profile' : 'BaseIPythonApplication.profile',
63 63 'ipython-dir' : 'BaseIPythonApplication.ipython_dir',
64 64 'log-level' : 'Application.log_level',
65 65 'config' : 'BaseIPythonApplication.extra_config_file',
66 66 }
67 67
68 68 base_flags = dict(
69 69 debug = ({'Application' : {'log_level' : logging.DEBUG}},
70 70 "set log level to logging.DEBUG (maximize logging output)"),
71 71 quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
72 72 "set log level to logging.CRITICAL (minimize logging output)"),
73 73 init = ({'BaseIPythonApplication' : {
74 74 'copy_config_files' : True,
75 75 'auto_create' : True}
76 76 }, """Initialize profile with default config files. This is equivalent
77 77 to running `ipython profile create <profile>` prior to startup.
78 78 """)
79 79 )
80 80
81 81 class ProfileAwareConfigLoader(PyFileConfigLoader):
82 82 """A Python file config loader that is aware of IPython profiles."""
83 83 def load_subconfig(self, fname, path=None, profile=None):
84 84 if profile is not None:
85 85 try:
86 86 profile_dir = ProfileDir.find_profile_dir_by_name(
87 87 get_ipython_dir(),
88 88 profile,
89 89 )
90 90 except ProfileDirError:
91 91 return
92 92 path = profile_dir.location
93 93 return super(ProfileAwareConfigLoader, self).load_subconfig(fname, path=path)
94 94
95 95 class BaseIPythonApplication(Application):
96 96
97 97 name = Unicode(u'ipython')
98 98 description = Unicode(u'IPython: an enhanced interactive Python shell.')
99 99 version = Unicode(release.version)
100 100
101 101 aliases = Dict(base_aliases)
102 102 flags = Dict(base_flags)
103 103 classes = List([ProfileDir])
104 104
105 105 # enable `load_subconfig('cfg.py', profile='name')`
106 106 python_config_loader_class = ProfileAwareConfigLoader
107 107
108 108 # Track whether the config_file has changed,
109 109 # because some logic happens only if we aren't using the default.
110 110 config_file_specified = Set()
111 111
112 112 config_file_name = Unicode()
113 113 @default('config_file_name')
114 114 def _config_file_name_default(self):
115 115 return self.name.replace('-','_') + u'_config.py'
116 116 @observe('config_file_name')
117 117 def _config_file_name_changed(self, change):
118 118 if change['new'] != change['old']:
119 119 self.config_file_specified.add(change['new'])
120 120
121 121 # The directory that contains IPython's builtin profiles.
122 122 builtin_profile_dir = Unicode(
123 123 os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
124 124 )
125 125
126 126 config_file_paths = List(Unicode())
127 127 @default('config_file_paths')
128 128 def _config_file_paths_default(self):
129 return [py3compat.getcwd()]
129 return [os.getcwd()]
130 130
131 131 extra_config_file = Unicode(
132 132 help="""Path to an extra config file to load.
133 133
134 134 If specified, load this config file in addition to any other IPython config.
135 135 """).tag(config=True)
136 136 @observe('extra_config_file')
137 137 def _extra_config_file_changed(self, change):
138 138 old = change['old']
139 139 new = change['new']
140 140 try:
141 141 self.config_files.remove(old)
142 142 except ValueError:
143 143 pass
144 144 self.config_file_specified.add(new)
145 145 self.config_files.append(new)
146 146
147 147 profile = Unicode(u'default',
148 148 help="""The IPython profile to use."""
149 149 ).tag(config=True)
150 150
151 151 @observe('profile')
152 152 def _profile_changed(self, change):
153 153 self.builtin_profile_dir = os.path.join(
154 154 get_ipython_package_dir(), u'config', u'profile', change['new']
155 155 )
156 156
157 157 ipython_dir = Unicode(
158 158 help="""
159 159 The name of the IPython directory. This directory is used for logging
160 160 configuration (through profiles), history storage, etc. The default
161 161 is usually $HOME/.ipython. This option can also be specified through
162 162 the environment variable IPYTHONDIR.
163 163 """
164 164 ).tag(config=True)
165 165 @default('ipython_dir')
166 166 def _ipython_dir_default(self):
167 167 d = get_ipython_dir()
168 168 self._ipython_dir_changed({
169 169 'name': 'ipython_dir',
170 170 'old': d,
171 171 'new': d,
172 172 })
173 173 return d
174 174
175 175 _in_init_profile_dir = False
176 176 profile_dir = Instance(ProfileDir, allow_none=True)
177 177 @default('profile_dir')
178 178 def _profile_dir_default(self):
179 179 # avoid recursion
180 180 if self._in_init_profile_dir:
181 181 return
182 182 # profile_dir requested early, force initialization
183 183 self.init_profile_dir()
184 184 return self.profile_dir
185 185
186 186 overwrite = Bool(False,
187 187 help="""Whether to overwrite existing config files when copying"""
188 188 ).tag(config=True)
189 189 auto_create = Bool(False,
190 190 help="""Whether to create profile dir if it doesn't exist"""
191 191 ).tag(config=True)
192 192
193 193 config_files = List(Unicode())
194 194 @default('config_files')
195 195 def _config_files_default(self):
196 196 return [self.config_file_name]
197 197
198 198 copy_config_files = Bool(False,
199 199 help="""Whether to install the default config files into the profile dir.
200 200 If a new profile is being created, and IPython contains config files for that
201 201 profile, then they will be staged into the new directory. Otherwise,
202 202 default config files will be automatically generated.
203 203 """).tag(config=True)
204 204
205 205 verbose_crash = Bool(False,
206 206 help="""Create a massive crash report when IPython encounters what may be an
207 207 internal error. The default is to append a short message to the
208 208 usual traceback""").tag(config=True)
209 209
210 210 # The class to use as the crash handler.
211 211 crash_handler_class = Type(crashhandler.CrashHandler)
212 212
213 213 @catch_config_error
214 214 def __init__(self, **kwargs):
215 215 super(BaseIPythonApplication, self).__init__(**kwargs)
216 216 # ensure current working directory exists
217 217 try:
218 py3compat.getcwd()
218 os.getcwd()
219 219 except:
220 220 # exit if cwd doesn't exist
221 221 self.log.error("Current working directory doesn't exist.")
222 222 self.exit(1)
223 223
224 224 #-------------------------------------------------------------------------
225 225 # Various stages of Application creation
226 226 #-------------------------------------------------------------------------
227 227
228 228 deprecated_subcommands = {}
229 229
230 230 def initialize_subcommand(self, subc, argv=None):
231 231 if subc in self.deprecated_subcommands:
232 232 self.log.warning("Subcommand `ipython {sub}` is deprecated and will be removed "
233 233 "in future versions.".format(sub=subc))
234 234 self.log.warning("You likely want to use `jupyter {sub}` in the "
235 235 "future".format(sub=subc))
236 236 return super(BaseIPythonApplication, self).initialize_subcommand(subc, argv)
237 237
238 238 def init_crash_handler(self):
239 239 """Create a crash handler, typically setting sys.excepthook to it."""
240 240 self.crash_handler = self.crash_handler_class(self)
241 241 sys.excepthook = self.excepthook
242 242 def unset_crashhandler():
243 243 sys.excepthook = sys.__excepthook__
244 244 atexit.register(unset_crashhandler)
245 245
246 246 def excepthook(self, etype, evalue, tb):
247 247 """this is sys.excepthook after init_crashhandler
248 248
249 249 set self.verbose_crash=True to use our full crashhandler, instead of
250 250 a regular traceback with a short message (crash_handler_lite)
251 251 """
252 252
253 253 if self.verbose_crash:
254 254 return self.crash_handler(etype, evalue, tb)
255 255 else:
256 256 return crashhandler.crash_handler_lite(etype, evalue, tb)
257 257
258 258 @observe('ipython_dir')
259 259 def _ipython_dir_changed(self, change):
260 260 old = change['old']
261 261 new = change['new']
262 262 if old is not Undefined:
263 263 str_old = py3compat.cast_bytes_py2(os.path.abspath(old),
264 264 sys.getfilesystemencoding()
265 265 )
266 266 if str_old in sys.path:
267 267 sys.path.remove(str_old)
268 268 str_path = py3compat.cast_bytes_py2(os.path.abspath(new),
269 269 sys.getfilesystemencoding()
270 270 )
271 271 sys.path.append(str_path)
272 272 ensure_dir_exists(new)
273 273 readme = os.path.join(new, 'README')
274 274 readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README')
275 275 if not os.path.exists(readme) and os.path.exists(readme_src):
276 276 shutil.copy(readme_src, readme)
277 277 for d in ('extensions', 'nbextensions'):
278 278 path = os.path.join(new, d)
279 279 try:
280 280 ensure_dir_exists(path)
281 281 except OSError as e:
282 282 # this will not be EEXIST
283 283 self.log.error("couldn't create path %s: %s", path, e)
284 284 self.log.debug("IPYTHONDIR set to: %s" % new)
285 285
286 286 def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS):
287 287 """Load the config file.
288 288
289 289 By default, errors in loading config are handled, and a warning
290 290 printed on screen. For testing, the suppress_errors option is set
291 291 to False, so errors will make tests fail.
292 292
293 293 `supress_errors` default value is to be `None` in which case the
294 294 behavior default to the one of `traitlets.Application`.
295 295
296 296 The default value can be set :
297 297 - to `False` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '0', or 'false' (case insensitive).
298 298 - to `True` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '1' or 'true' (case insensitive).
299 299 - to `None` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '' (empty string) or leaving it unset.
300 300
301 301 Any other value are invalid, and will make IPython exit with a non-zero return code.
302 302 """
303 303
304 304
305 305 self.log.debug("Searching path %s for config files", self.config_file_paths)
306 306 base_config = 'ipython_config.py'
307 307 self.log.debug("Attempting to load config file: %s" %
308 308 base_config)
309 309 try:
310 310 if suppress_errors is not None:
311 311 old_value = Application.raise_config_file_errors
312 312 Application.raise_config_file_errors = not suppress_errors;
313 313 Application.load_config_file(
314 314 self,
315 315 base_config,
316 316 path=self.config_file_paths
317 317 )
318 318 except ConfigFileNotFound:
319 319 # ignore errors loading parent
320 320 self.log.debug("Config file %s not found", base_config)
321 321 pass
322 322 if suppress_errors is not None:
323 323 Application.raise_config_file_errors = old_value
324 324
325 325 for config_file_name in self.config_files:
326 326 if not config_file_name or config_file_name == base_config:
327 327 continue
328 328 self.log.debug("Attempting to load config file: %s" %
329 329 self.config_file_name)
330 330 try:
331 331 Application.load_config_file(
332 332 self,
333 333 config_file_name,
334 334 path=self.config_file_paths
335 335 )
336 336 except ConfigFileNotFound:
337 337 # Only warn if the default config file was NOT being used.
338 338 if config_file_name in self.config_file_specified:
339 339 msg = self.log.warning
340 340 else:
341 341 msg = self.log.debug
342 342 msg("Config file not found, skipping: %s", config_file_name)
343 343 except Exception:
344 344 # For testing purposes.
345 345 if not suppress_errors:
346 346 raise
347 347 self.log.warning("Error loading config file: %s" %
348 348 self.config_file_name, exc_info=True)
349 349
350 350 def init_profile_dir(self):
351 351 """initialize the profile dir"""
352 352 self._in_init_profile_dir = True
353 353 if self.profile_dir is not None:
354 354 # already ran
355 355 return
356 356 if 'ProfileDir.location' not in self.config:
357 357 # location not specified, find by profile name
358 358 try:
359 359 p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
360 360 except ProfileDirError:
361 361 # not found, maybe create it (always create default profile)
362 362 if self.auto_create or self.profile == 'default':
363 363 try:
364 364 p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
365 365 except ProfileDirError:
366 366 self.log.fatal("Could not create profile: %r"%self.profile)
367 367 self.exit(1)
368 368 else:
369 369 self.log.info("Created profile dir: %r"%p.location)
370 370 else:
371 371 self.log.fatal("Profile %r not found."%self.profile)
372 372 self.exit(1)
373 373 else:
374 374 self.log.debug("Using existing profile dir: %r"%p.location)
375 375 else:
376 376 location = self.config.ProfileDir.location
377 377 # location is fully specified
378 378 try:
379 379 p = ProfileDir.find_profile_dir(location, self.config)
380 380 except ProfileDirError:
381 381 # not found, maybe create it
382 382 if self.auto_create:
383 383 try:
384 384 p = ProfileDir.create_profile_dir(location, self.config)
385 385 except ProfileDirError:
386 386 self.log.fatal("Could not create profile directory: %r"%location)
387 387 self.exit(1)
388 388 else:
389 389 self.log.debug("Creating new profile dir: %r"%location)
390 390 else:
391 391 self.log.fatal("Profile directory %r not found."%location)
392 392 self.exit(1)
393 393 else:
394 394 self.log.info("Using existing profile dir: %r"%location)
395 395 # if profile_dir is specified explicitly, set profile name
396 396 dir_name = os.path.basename(p.location)
397 397 if dir_name.startswith('profile_'):
398 398 self.profile = dir_name[8:]
399 399
400 400 self.profile_dir = p
401 401 self.config_file_paths.append(p.location)
402 402 self._in_init_profile_dir = False
403 403
404 404 def init_config_files(self):
405 405 """[optionally] copy default config files into profile dir."""
406 406 self.config_file_paths.extend(SYSTEM_CONFIG_DIRS)
407 407 # copy config files
408 408 path = self.builtin_profile_dir
409 409 if self.copy_config_files:
410 410 src = self.profile
411 411
412 412 cfg = self.config_file_name
413 413 if path and os.path.exists(os.path.join(path, cfg)):
414 414 self.log.warning("Staging %r from %s into %r [overwrite=%s]"%(
415 415 cfg, src, self.profile_dir.location, self.overwrite)
416 416 )
417 417 self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
418 418 else:
419 419 self.stage_default_config_file()
420 420 else:
421 421 # Still stage *bundled* config files, but not generated ones
422 422 # This is necessary for `ipython profile=sympy` to load the profile
423 423 # on the first go
424 424 files = glob.glob(os.path.join(path, '*.py'))
425 425 for fullpath in files:
426 426 cfg = os.path.basename(fullpath)
427 427 if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
428 428 # file was copied
429 429 self.log.warning("Staging bundled %s from %s into %r"%(
430 430 cfg, self.profile, self.profile_dir.location)
431 431 )
432 432
433 433
434 434 def stage_default_config_file(self):
435 435 """auto generate default config file, and stage it into the profile."""
436 436 s = self.generate_config_file()
437 437 fname = os.path.join(self.profile_dir.location, self.config_file_name)
438 438 if self.overwrite or not os.path.exists(fname):
439 439 self.log.warning("Generating default config file: %r"%(fname))
440 440 with open(fname, 'w') as f:
441 441 f.write(s)
442 442
443 443 @catch_config_error
444 444 def initialize(self, argv=None):
445 445 # don't hook up crash handler before parsing command-line
446 446 self.parse_command_line(argv)
447 447 self.init_crash_handler()
448 448 if self.subapp is not None:
449 449 # stop here if subapp is taking over
450 450 return
451 451 # save a copy of CLI config to re-load after config files
452 452 # so that it has highest priority
453 453 cl_config = deepcopy(self.config)
454 454 self.init_profile_dir()
455 455 self.init_config_files()
456 456 self.load_config_file()
457 457 # enforce cl-opts override configfile opts:
458 458 self.update_config(cl_config)
@@ -1,910 +1,910 b''
1 1 """ History related magics and functionality """
2 2
3 3 # Copyright (c) IPython Development Team.
4 4 # Distributed under the terms of the Modified BSD License.
5 5
6 6
7 7 import atexit
8 8 import datetime
9 9 import os
10 10 import re
11 11 try:
12 12 import sqlite3
13 13 except ImportError:
14 14 try:
15 15 from pysqlite2 import dbapi2 as sqlite3
16 16 except ImportError:
17 17 sqlite3 = None
18 18 import threading
19 19
20 20 from traitlets.config.configurable import LoggingConfigurable
21 21 from decorator import decorator
22 22 from IPython.utils.decorators import undoc
23 23 from IPython.utils.path import locate_profile
24 24 from IPython.utils import py3compat
25 25 from traitlets import (
26 26 Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
27 27 default, observe,
28 28 )
29 29 from warnings import warn
30 30
31 31 #-----------------------------------------------------------------------------
32 32 # Classes and functions
33 33 #-----------------------------------------------------------------------------
34 34
35 35 @undoc
36 36 class DummyDB(object):
37 37 """Dummy DB that will act as a black hole for history.
38 38
39 39 Only used in the absence of sqlite"""
40 40 def execute(*args, **kwargs):
41 41 return []
42 42
43 43 def commit(self, *args, **kwargs):
44 44 pass
45 45
46 46 def __enter__(self, *args, **kwargs):
47 47 pass
48 48
49 49 def __exit__(self, *args, **kwargs):
50 50 pass
51 51
52 52
53 53 @decorator
54 54 def needs_sqlite(f, self, *a, **kw):
55 55 """Decorator: return an empty list in the absence of sqlite."""
56 56 if sqlite3 is None or not self.enabled:
57 57 return []
58 58 else:
59 59 return f(self, *a, **kw)
60 60
61 61
62 62 if sqlite3 is not None:
63 63 DatabaseError = sqlite3.DatabaseError
64 64 OperationalError = sqlite3.OperationalError
65 65 else:
66 66 @undoc
67 67 class DatabaseError(Exception):
68 68 "Dummy exception when sqlite could not be imported. Should never occur."
69 69
70 70 @undoc
71 71 class OperationalError(Exception):
72 72 "Dummy exception when sqlite could not be imported. Should never occur."
73 73
74 74 # use 16kB as threshold for whether a corrupt history db should be saved
75 75 # that should be at least 100 entries or so
76 76 _SAVE_DB_SIZE = 16384
77 77
78 78 @decorator
79 79 def catch_corrupt_db(f, self, *a, **kw):
80 80 """A decorator which wraps HistoryAccessor method calls to catch errors from
81 81 a corrupt SQLite database, move the old database out of the way, and create
82 82 a new one.
83 83
84 84 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
85 85 not just a corrupt file.
86 86 """
87 87 try:
88 88 return f(self, *a, **kw)
89 89 except (DatabaseError, OperationalError) as e:
90 90 self._corrupt_db_counter += 1
91 91 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
92 92 if self.hist_file != ':memory:':
93 93 if self._corrupt_db_counter > self._corrupt_db_limit:
94 94 self.hist_file = ':memory:'
95 95 self.log.error("Failed to load history too many times, history will not be saved.")
96 96 elif os.path.isfile(self.hist_file):
97 97 # move the file out of the way
98 98 base, ext = os.path.splitext(self.hist_file)
99 99 size = os.stat(self.hist_file).st_size
100 100 if size >= _SAVE_DB_SIZE:
101 101 # if there's significant content, avoid clobbering
102 102 now = datetime.datetime.now().isoformat().replace(':', '.')
103 103 newpath = base + '-corrupt-' + now + ext
104 104 # don't clobber previous corrupt backups
105 105 for i in range(100):
106 106 if not os.path.isfile(newpath):
107 107 break
108 108 else:
109 109 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
110 110 else:
111 111 # not much content, possibly empty; don't worry about clobbering
112 112 # maybe we should just delete it?
113 113 newpath = base + '-corrupt' + ext
114 114 os.rename(self.hist_file, newpath)
115 115 self.log.error("History file was moved to %s and a new file created.", newpath)
116 116 self.init_db()
117 117 return []
118 118 else:
119 119 # Failed with :memory:, something serious is wrong
120 120 raise
121 121
122 122 class HistoryAccessorBase(LoggingConfigurable):
123 123 """An abstract class for History Accessors """
124 124
125 125 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
126 126 raise NotImplementedError
127 127
128 128 def search(self, pattern="*", raw=True, search_raw=True,
129 129 output=False, n=None, unique=False):
130 130 raise NotImplementedError
131 131
132 132 def get_range(self, session, start=1, stop=None, raw=True,output=False):
133 133 raise NotImplementedError
134 134
135 135 def get_range_by_str(self, rangestr, raw=True, output=False):
136 136 raise NotImplementedError
137 137
138 138
139 139 class HistoryAccessor(HistoryAccessorBase):
140 140 """Access the history database without adding to it.
141 141
142 142 This is intended for use by standalone history tools. IPython shells use
143 143 HistoryManager, below, which is a subclass of this."""
144 144
145 145 # counter for init_db retries, so we don't keep trying over and over
146 146 _corrupt_db_counter = 0
147 147 # after two failures, fallback on :memory:
148 148 _corrupt_db_limit = 2
149 149
150 150 # String holding the path to the history file
151 151 hist_file = Unicode(
152 152 help="""Path to file to use for SQLite history database.
153 153
154 154 By default, IPython will put the history database in the IPython
155 155 profile directory. If you would rather share one history among
156 156 profiles, you can set this value in each, so that they are consistent.
157 157
158 158 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
159 159 mounts. If you see IPython hanging, try setting this to something on a
160 160 local disk, e.g::
161 161
162 162 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
163 163
164 164 you can also use the specific value `:memory:` (including the colon
165 165 at both end but not the back ticks), to avoid creating an history file.
166 166
167 167 """).tag(config=True)
168 168
169 169 enabled = Bool(True,
170 170 help="""enable the SQLite history
171 171
172 172 set enabled=False to disable the SQLite history,
173 173 in which case there will be no stored history, no SQLite connection,
174 174 and no background saving thread. This may be necessary in some
175 175 threaded environments where IPython is embedded.
176 176 """
177 177 ).tag(config=True)
178 178
179 179 connection_options = Dict(
180 180 help="""Options for configuring the SQLite connection
181 181
182 182 These options are passed as keyword args to sqlite3.connect
183 183 when establishing database conenctions.
184 184 """
185 185 ).tag(config=True)
186 186
187 187 # The SQLite database
188 188 db = Any()
189 189 @observe('db')
190 190 def _db_changed(self, change):
191 191 """validate the db, since it can be an Instance of two different types"""
192 192 new = change['new']
193 193 connection_types = (DummyDB,)
194 194 if sqlite3 is not None:
195 195 connection_types = (DummyDB, sqlite3.Connection)
196 196 if not isinstance(new, connection_types):
197 197 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
198 198 (self.__class__.__name__, new)
199 199 raise TraitError(msg)
200 200
201 201 def __init__(self, profile='default', hist_file=u'', **traits):
202 202 """Create a new history accessor.
203 203
204 204 Parameters
205 205 ----------
206 206 profile : str
207 207 The name of the profile from which to open history.
208 208 hist_file : str
209 209 Path to an SQLite history database stored by IPython. If specified,
210 210 hist_file overrides profile.
211 211 config : :class:`~traitlets.config.loader.Config`
212 212 Config object. hist_file can also be set through this.
213 213 """
214 214 # We need a pointer back to the shell for various tasks.
215 215 super(HistoryAccessor, self).__init__(**traits)
216 216 # defer setting hist_file from kwarg until after init,
217 217 # otherwise the default kwarg value would clobber any value
218 218 # set by config
219 219 if hist_file:
220 220 self.hist_file = hist_file
221 221
222 222 if self.hist_file == u'':
223 223 # No one has set the hist_file, yet.
224 224 self.hist_file = self._get_hist_file_name(profile)
225 225
226 226 if sqlite3 is None and self.enabled:
227 227 warn("IPython History requires SQLite, your history will not be saved")
228 228 self.enabled = False
229 229
230 230 self.init_db()
231 231
232 232 def _get_hist_file_name(self, profile='default'):
233 233 """Find the history file for the given profile name.
234 234
235 235 This is overridden by the HistoryManager subclass, to use the shell's
236 236 active profile.
237 237
238 238 Parameters
239 239 ----------
240 240 profile : str
241 241 The name of a profile which has a history file.
242 242 """
243 243 return os.path.join(locate_profile(profile), 'history.sqlite')
244 244
245 245 @catch_corrupt_db
246 246 def init_db(self):
247 247 """Connect to the database, and create tables if necessary."""
248 248 if not self.enabled:
249 249 self.db = DummyDB()
250 250 return
251 251
252 252 # use detect_types so that timestamps return datetime objects
253 253 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
254 254 kwargs.update(self.connection_options)
255 255 self.db = sqlite3.connect(self.hist_file, **kwargs)
256 256 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
257 257 primary key autoincrement, start timestamp,
258 258 end timestamp, num_cmds integer, remark text)""")
259 259 self.db.execute("""CREATE TABLE IF NOT EXISTS history
260 260 (session integer, line integer, source text, source_raw text,
261 261 PRIMARY KEY (session, line))""")
262 262 # Output history is optional, but ensure the table's there so it can be
263 263 # enabled later.
264 264 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
265 265 (session integer, line integer, output text,
266 266 PRIMARY KEY (session, line))""")
267 267 self.db.commit()
268 268 # success! reset corrupt db count
269 269 self._corrupt_db_counter = 0
270 270
271 271 def writeout_cache(self):
272 272 """Overridden by HistoryManager to dump the cache before certain
273 273 database lookups."""
274 274 pass
275 275
276 276 ## -------------------------------
277 277 ## Methods for retrieving history:
278 278 ## -------------------------------
279 279 def _run_sql(self, sql, params, raw=True, output=False):
280 280 """Prepares and runs an SQL query for the history database.
281 281
282 282 Parameters
283 283 ----------
284 284 sql : str
285 285 Any filtering expressions to go after SELECT ... FROM ...
286 286 params : tuple
287 287 Parameters passed to the SQL query (to replace "?")
288 288 raw, output : bool
289 289 See :meth:`get_range`
290 290
291 291 Returns
292 292 -------
293 293 Tuples as :meth:`get_range`
294 294 """
295 295 toget = 'source_raw' if raw else 'source'
296 296 sqlfrom = "history"
297 297 if output:
298 298 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
299 299 toget = "history.%s, output_history.output" % toget
300 300 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
301 301 (toget, sqlfrom) + sql, params)
302 302 if output: # Regroup into 3-tuples, and parse JSON
303 303 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
304 304 return cur
305 305
306 306 @needs_sqlite
307 307 @catch_corrupt_db
308 308 def get_session_info(self, session):
309 309 """Get info about a session.
310 310
311 311 Parameters
312 312 ----------
313 313
314 314 session : int
315 315 Session number to retrieve.
316 316
317 317 Returns
318 318 -------
319 319
320 320 session_id : int
321 321 Session ID number
322 322 start : datetime
323 323 Timestamp for the start of the session.
324 324 end : datetime
325 325 Timestamp for the end of the session, or None if IPython crashed.
326 326 num_cmds : int
327 327 Number of commands run, or None if IPython crashed.
328 328 remark : unicode
329 329 A manually set description.
330 330 """
331 331 query = "SELECT * from sessions where session == ?"
332 332 return self.db.execute(query, (session,)).fetchone()
333 333
334 334 @catch_corrupt_db
335 335 def get_last_session_id(self):
336 336 """Get the last session ID currently in the database.
337 337
338 338 Within IPython, this should be the same as the value stored in
339 339 :attr:`HistoryManager.session_number`.
340 340 """
341 341 for record in self.get_tail(n=1, include_latest=True):
342 342 return record[0]
343 343
344 344 @catch_corrupt_db
345 345 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
346 346 """Get the last n lines from the history database.
347 347
348 348 Parameters
349 349 ----------
350 350 n : int
351 351 The number of lines to get
352 352 raw, output : bool
353 353 See :meth:`get_range`
354 354 include_latest : bool
355 355 If False (default), n+1 lines are fetched, and the latest one
356 356 is discarded. This is intended to be used where the function
357 357 is called by a user command, which it should not return.
358 358
359 359 Returns
360 360 -------
361 361 Tuples as :meth:`get_range`
362 362 """
363 363 self.writeout_cache()
364 364 if not include_latest:
365 365 n += 1
366 366 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
367 367 (n,), raw=raw, output=output)
368 368 if not include_latest:
369 369 return reversed(list(cur)[1:])
370 370 return reversed(list(cur))
371 371
372 372 @catch_corrupt_db
373 373 def search(self, pattern="*", raw=True, search_raw=True,
374 374 output=False, n=None, unique=False):
375 375 """Search the database using unix glob-style matching (wildcards
376 376 * and ?).
377 377
378 378 Parameters
379 379 ----------
380 380 pattern : str
381 381 The wildcarded pattern to match when searching
382 382 search_raw : bool
383 383 If True, search the raw input, otherwise, the parsed input
384 384 raw, output : bool
385 385 See :meth:`get_range`
386 386 n : None or int
387 387 If an integer is given, it defines the limit of
388 388 returned entries.
389 389 unique : bool
390 390 When it is true, return only unique entries.
391 391
392 392 Returns
393 393 -------
394 394 Tuples as :meth:`get_range`
395 395 """
396 396 tosearch = "source_raw" if search_raw else "source"
397 397 if output:
398 398 tosearch = "history." + tosearch
399 399 self.writeout_cache()
400 400 sqlform = "WHERE %s GLOB ?" % tosearch
401 401 params = (pattern,)
402 402 if unique:
403 403 sqlform += ' GROUP BY {0}'.format(tosearch)
404 404 if n is not None:
405 405 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
406 406 params += (n,)
407 407 elif unique:
408 408 sqlform += " ORDER BY session, line"
409 409 cur = self._run_sql(sqlform, params, raw=raw, output=output)
410 410 if n is not None:
411 411 return reversed(list(cur))
412 412 return cur
413 413
414 414 @catch_corrupt_db
415 415 def get_range(self, session, start=1, stop=None, raw=True,output=False):
416 416 """Retrieve input by session.
417 417
418 418 Parameters
419 419 ----------
420 420 session : int
421 421 Session number to retrieve.
422 422 start : int
423 423 First line to retrieve.
424 424 stop : int
425 425 End of line range (excluded from output itself). If None, retrieve
426 426 to the end of the session.
427 427 raw : bool
428 428 If True, return untranslated input
429 429 output : bool
430 430 If True, attempt to include output. This will be 'real' Python
431 431 objects for the current session, or text reprs from previous
432 432 sessions if db_log_output was enabled at the time. Where no output
433 433 is found, None is used.
434 434
435 435 Returns
436 436 -------
437 437 entries
438 438 An iterator over the desired lines. Each line is a 3-tuple, either
439 439 (session, line, input) if output is False, or
440 440 (session, line, (input, output)) if output is True.
441 441 """
442 442 if stop:
443 443 lineclause = "line >= ? AND line < ?"
444 444 params = (session, start, stop)
445 445 else:
446 446 lineclause = "line>=?"
447 447 params = (session, start)
448 448
449 449 return self._run_sql("WHERE session==? AND %s" % lineclause,
450 450 params, raw=raw, output=output)
451 451
452 452 def get_range_by_str(self, rangestr, raw=True, output=False):
453 453 """Get lines of history from a string of ranges, as used by magic
454 454 commands %hist, %save, %macro, etc.
455 455
456 456 Parameters
457 457 ----------
458 458 rangestr : str
459 459 A string specifying ranges, e.g. "5 ~2/1-4". See
460 460 :func:`magic_history` for full details.
461 461 raw, output : bool
462 462 As :meth:`get_range`
463 463
464 464 Returns
465 465 -------
466 466 Tuples as :meth:`get_range`
467 467 """
468 468 for sess, s, e in extract_hist_ranges(rangestr):
469 469 for line in self.get_range(sess, s, e, raw=raw, output=output):
470 470 yield line
471 471
472 472
473 473 class HistoryManager(HistoryAccessor):
474 474 """A class to organize all history-related functionality in one place.
475 475 """
476 476 # Public interface
477 477
478 478 # An instance of the IPython shell we are attached to
479 479 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
480 480 allow_none=True)
481 481 # Lists to hold processed and raw history. These start with a blank entry
482 482 # so that we can index them starting from 1
483 483 input_hist_parsed = List([""])
484 484 input_hist_raw = List([""])
485 485 # A list of directories visited during session
486 486 dir_hist = List()
487 487 @default('dir_hist')
488 488 def _dir_hist_default(self):
489 489 try:
490 return [py3compat.getcwd()]
490 return [os.getcwd()]
491 491 except OSError:
492 492 return []
493 493
494 494 # A dict of output history, keyed with ints from the shell's
495 495 # execution count.
496 496 output_hist = Dict()
497 497 # The text/plain repr of outputs.
498 498 output_hist_reprs = Dict()
499 499
500 500 # The number of the current session in the history database
501 501 session_number = Integer()
502 502
503 503 db_log_output = Bool(False,
504 504 help="Should the history database include output? (default: no)"
505 505 ).tag(config=True)
506 506 db_cache_size = Integer(0,
507 507 help="Write to database every x commands (higher values save disk access & power).\n"
508 508 "Values of 1 or less effectively disable caching."
509 509 ).tag(config=True)
510 510 # The input and output caches
511 511 db_input_cache = List()
512 512 db_output_cache = List()
513 513
514 514 # History saving in separate thread
515 515 save_thread = Instance('IPython.core.history.HistorySavingThread',
516 516 allow_none=True)
517 517 try: # Event is a function returning an instance of _Event...
518 518 save_flag = Instance(threading._Event, allow_none=True)
519 519 except AttributeError: # ...until Python 3.3, when it's a class.
520 520 save_flag = Instance(threading.Event, allow_none=True)
521 521
522 522 # Private interface
523 523 # Variables used to store the three last inputs from the user. On each new
524 524 # history update, we populate the user's namespace with these, shifted as
525 525 # necessary.
526 526 _i00 = Unicode(u'')
527 527 _i = Unicode(u'')
528 528 _ii = Unicode(u'')
529 529 _iii = Unicode(u'')
530 530
531 531 # A regex matching all forms of the exit command, so that we don't store
532 532 # them in the history (it's annoying to rewind the first entry and land on
533 533 # an exit call).
534 534 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
535 535
536 536 def __init__(self, shell=None, config=None, **traits):
537 537 """Create a new history manager associated with a shell instance.
538 538 """
539 539 # We need a pointer back to the shell for various tasks.
540 540 super(HistoryManager, self).__init__(shell=shell, config=config,
541 541 **traits)
542 542 self.save_flag = threading.Event()
543 543 self.db_input_cache_lock = threading.Lock()
544 544 self.db_output_cache_lock = threading.Lock()
545 545
546 546 try:
547 547 self.new_session()
548 548 except OperationalError:
549 549 self.log.error("Failed to create history session in %s. History will not be saved.",
550 550 self.hist_file, exc_info=True)
551 551 self.hist_file = ':memory:'
552 552
553 553 if self.enabled and self.hist_file != ':memory:':
554 554 self.save_thread = HistorySavingThread(self)
555 555 self.save_thread.start()
556 556
557 557 def _get_hist_file_name(self, profile=None):
558 558 """Get default history file name based on the Shell's profile.
559 559
560 560 The profile parameter is ignored, but must exist for compatibility with
561 561 the parent class."""
562 562 profile_dir = self.shell.profile_dir.location
563 563 return os.path.join(profile_dir, 'history.sqlite')
564 564
565 565 @needs_sqlite
566 566 def new_session(self, conn=None):
567 567 """Get a new session number."""
568 568 if conn is None:
569 569 conn = self.db
570 570
571 571 with conn:
572 572 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
573 573 NULL, "") """, (datetime.datetime.now(),))
574 574 self.session_number = cur.lastrowid
575 575
576 576 def end_session(self):
577 577 """Close the database session, filling in the end time and line count."""
578 578 self.writeout_cache()
579 579 with self.db:
580 580 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
581 581 session==?""", (datetime.datetime.now(),
582 582 len(self.input_hist_parsed)-1, self.session_number))
583 583 self.session_number = 0
584 584
585 585 def name_session(self, name):
586 586 """Give the current session a name in the history database."""
587 587 with self.db:
588 588 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
589 589 (name, self.session_number))
590 590
591 591 def reset(self, new_session=True):
592 592 """Clear the session history, releasing all object references, and
593 593 optionally open a new session."""
594 594 self.output_hist.clear()
595 595 # The directory history can't be completely empty
596 self.dir_hist[:] = [py3compat.getcwd()]
596 self.dir_hist[:] = [os.getcwd()]
597 597
598 598 if new_session:
599 599 if self.session_number:
600 600 self.end_session()
601 601 self.input_hist_parsed[:] = [""]
602 602 self.input_hist_raw[:] = [""]
603 603 self.new_session()
604 604
605 605 # ------------------------------
606 606 # Methods for retrieving history
607 607 # ------------------------------
608 608 def get_session_info(self, session=0):
609 609 """Get info about a session.
610 610
611 611 Parameters
612 612 ----------
613 613
614 614 session : int
615 615 Session number to retrieve. The current session is 0, and negative
616 616 numbers count back from current session, so -1 is the previous session.
617 617
618 618 Returns
619 619 -------
620 620
621 621 session_id : int
622 622 Session ID number
623 623 start : datetime
624 624 Timestamp for the start of the session.
625 625 end : datetime
626 626 Timestamp for the end of the session, or None if IPython crashed.
627 627 num_cmds : int
628 628 Number of commands run, or None if IPython crashed.
629 629 remark : unicode
630 630 A manually set description.
631 631 """
632 632 if session <= 0:
633 633 session += self.session_number
634 634
635 635 return super(HistoryManager, self).get_session_info(session=session)
636 636
637 637 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
638 638 """Get input and output history from the current session. Called by
639 639 get_range, and takes similar parameters."""
640 640 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
641 641
642 642 n = len(input_hist)
643 643 if start < 0:
644 644 start += n
645 645 if not stop or (stop > n):
646 646 stop = n
647 647 elif stop < 0:
648 648 stop += n
649 649
650 650 for i in range(start, stop):
651 651 if output:
652 652 line = (input_hist[i], self.output_hist_reprs.get(i))
653 653 else:
654 654 line = input_hist[i]
655 655 yield (0, i, line)
656 656
657 657 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
658 658 """Retrieve input by session.
659 659
660 660 Parameters
661 661 ----------
662 662 session : int
663 663 Session number to retrieve. The current session is 0, and negative
664 664 numbers count back from current session, so -1 is previous session.
665 665 start : int
666 666 First line to retrieve.
667 667 stop : int
668 668 End of line range (excluded from output itself). If None, retrieve
669 669 to the end of the session.
670 670 raw : bool
671 671 If True, return untranslated input
672 672 output : bool
673 673 If True, attempt to include output. This will be 'real' Python
674 674 objects for the current session, or text reprs from previous
675 675 sessions if db_log_output was enabled at the time. Where no output
676 676 is found, None is used.
677 677
678 678 Returns
679 679 -------
680 680 entries
681 681 An iterator over the desired lines. Each line is a 3-tuple, either
682 682 (session, line, input) if output is False, or
683 683 (session, line, (input, output)) if output is True.
684 684 """
685 685 if session <= 0:
686 686 session += self.session_number
687 687 if session==self.session_number: # Current session
688 688 return self._get_range_session(start, stop, raw, output)
689 689 return super(HistoryManager, self).get_range(session, start, stop, raw,
690 690 output)
691 691
692 692 ## ----------------------------
693 693 ## Methods for storing history:
694 694 ## ----------------------------
695 695 def store_inputs(self, line_num, source, source_raw=None):
696 696 """Store source and raw input in history and create input cache
697 697 variables ``_i*``.
698 698
699 699 Parameters
700 700 ----------
701 701 line_num : int
702 702 The prompt number of this input.
703 703
704 704 source : str
705 705 Python input.
706 706
707 707 source_raw : str, optional
708 708 If given, this is the raw input without any IPython transformations
709 709 applied to it. If not given, ``source`` is used.
710 710 """
711 711 if source_raw is None:
712 712 source_raw = source
713 713 source = source.rstrip('\n')
714 714 source_raw = source_raw.rstrip('\n')
715 715
716 716 # do not store exit/quit commands
717 717 if self._exit_re.match(source_raw.strip()):
718 718 return
719 719
720 720 self.input_hist_parsed.append(source)
721 721 self.input_hist_raw.append(source_raw)
722 722
723 723 with self.db_input_cache_lock:
724 724 self.db_input_cache.append((line_num, source, source_raw))
725 725 # Trigger to flush cache and write to DB.
726 726 if len(self.db_input_cache) >= self.db_cache_size:
727 727 self.save_flag.set()
728 728
729 729 # update the auto _i variables
730 730 self._iii = self._ii
731 731 self._ii = self._i
732 732 self._i = self._i00
733 733 self._i00 = source_raw
734 734
735 735 # hackish access to user namespace to create _i1,_i2... dynamically
736 736 new_i = '_i%s' % line_num
737 737 to_main = {'_i': self._i,
738 738 '_ii': self._ii,
739 739 '_iii': self._iii,
740 740 new_i : self._i00 }
741 741
742 742 if self.shell is not None:
743 743 self.shell.push(to_main, interactive=False)
744 744
745 745 def store_output(self, line_num):
746 746 """If database output logging is enabled, this saves all the
747 747 outputs from the indicated prompt number to the database. It's
748 748 called by run_cell after code has been executed.
749 749
750 750 Parameters
751 751 ----------
752 752 line_num : int
753 753 The line number from which to save outputs
754 754 """
755 755 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
756 756 return
757 757 output = self.output_hist_reprs[line_num]
758 758
759 759 with self.db_output_cache_lock:
760 760 self.db_output_cache.append((line_num, output))
761 761 if self.db_cache_size <= 1:
762 762 self.save_flag.set()
763 763
764 764 def _writeout_input_cache(self, conn):
765 765 with conn:
766 766 for line in self.db_input_cache:
767 767 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
768 768 (self.session_number,)+line)
769 769
770 770 def _writeout_output_cache(self, conn):
771 771 with conn:
772 772 for line in self.db_output_cache:
773 773 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
774 774 (self.session_number,)+line)
775 775
776 776 @needs_sqlite
777 777 def writeout_cache(self, conn=None):
778 778 """Write any entries in the cache to the database."""
779 779 if conn is None:
780 780 conn = self.db
781 781
782 782 with self.db_input_cache_lock:
783 783 try:
784 784 self._writeout_input_cache(conn)
785 785 except sqlite3.IntegrityError:
786 786 self.new_session(conn)
787 787 print("ERROR! Session/line number was not unique in",
788 788 "database. History logging moved to new session",
789 789 self.session_number)
790 790 try:
791 791 # Try writing to the new session. If this fails, don't
792 792 # recurse
793 793 self._writeout_input_cache(conn)
794 794 except sqlite3.IntegrityError:
795 795 pass
796 796 finally:
797 797 self.db_input_cache = []
798 798
799 799 with self.db_output_cache_lock:
800 800 try:
801 801 self._writeout_output_cache(conn)
802 802 except sqlite3.IntegrityError:
803 803 print("!! Session/line number for output was not unique",
804 804 "in database. Output will not be stored.")
805 805 finally:
806 806 self.db_output_cache = []
807 807
808 808
809 809 class HistorySavingThread(threading.Thread):
810 810 """This thread takes care of writing history to the database, so that
811 811 the UI isn't held up while that happens.
812 812
813 813 It waits for the HistoryManager's save_flag to be set, then writes out
814 814 the history cache. The main thread is responsible for setting the flag when
815 815 the cache size reaches a defined threshold."""
816 816 daemon = True
817 817 stop_now = False
818 818 enabled = True
819 819 def __init__(self, history_manager):
820 820 super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
821 821 self.history_manager = history_manager
822 822 self.enabled = history_manager.enabled
823 823 atexit.register(self.stop)
824 824
825 825 @needs_sqlite
826 826 def run(self):
827 827 # We need a separate db connection per thread:
828 828 try:
829 829 self.db = sqlite3.connect(self.history_manager.hist_file,
830 830 **self.history_manager.connection_options
831 831 )
832 832 while True:
833 833 self.history_manager.save_flag.wait()
834 834 if self.stop_now:
835 835 self.db.close()
836 836 return
837 837 self.history_manager.save_flag.clear()
838 838 self.history_manager.writeout_cache(self.db)
839 839 except Exception as e:
840 840 print(("The history saving thread hit an unexpected error (%s)."
841 841 "History will not be written to the database.") % repr(e))
842 842
843 843 def stop(self):
844 844 """This can be called from the main thread to safely stop this thread.
845 845
846 846 Note that it does not attempt to write out remaining history before
847 847 exiting. That should be done by calling the HistoryManager's
848 848 end_session method."""
849 849 self.stop_now = True
850 850 self.history_manager.save_flag.set()
851 851 self.join()
852 852
853 853
854 854 # To match, e.g. ~5/8-~2/3
855 855 range_re = re.compile(r"""
856 856 ((?P<startsess>~?\d+)/)?
857 857 (?P<start>\d+)?
858 858 ((?P<sep>[\-:])
859 859 ((?P<endsess>~?\d+)/)?
860 860 (?P<end>\d+))?
861 861 $""", re.VERBOSE)
862 862
863 863
864 864 def extract_hist_ranges(ranges_str):
865 865 """Turn a string of history ranges into 3-tuples of (session, start, stop).
866 866
867 867 Examples
868 868 --------
869 869 >>> list(extract_hist_ranges("~8/5-~7/4 2"))
870 870 [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
871 871 """
872 872 for range_str in ranges_str.split():
873 873 rmatch = range_re.match(range_str)
874 874 if not rmatch:
875 875 continue
876 876 start = rmatch.group("start")
877 877 if start:
878 878 start = int(start)
879 879 end = rmatch.group("end")
880 880 # If no end specified, get (a, a + 1)
881 881 end = int(end) if end else start + 1
882 882 else: # start not specified
883 883 if not rmatch.group('startsess'): # no startsess
884 884 continue
885 885 start = 1
886 886 end = None # provide the entire session hist
887 887
888 888 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
889 889 end += 1
890 890 startsess = rmatch.group("startsess") or "0"
891 891 endsess = rmatch.group("endsess") or startsess
892 892 startsess = int(startsess.replace("~","-"))
893 893 endsess = int(endsess.replace("~","-"))
894 894 assert endsess >= startsess, "start session must be earlier than end session"
895 895
896 896 if endsess == startsess:
897 897 yield (startsess, start, end)
898 898 continue
899 899 # Multiple sessions in one range:
900 900 yield (startsess, start, None)
901 901 for sess in range(startsess+1, endsess):
902 902 yield (sess, 1, None)
903 903 yield (endsess, 1, end)
904 904
905 905
906 906 def _format_lineno(session, line):
907 907 """Helper function to format line numbers properly."""
908 908 if session == 0:
909 909 return str(line)
910 910 return "%s#%s" % (session, line)
@@ -1,3223 +1,3223 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Main IPython class."""
3 3
4 4 #-----------------------------------------------------------------------------
5 5 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
6 6 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
7 7 # Copyright (C) 2008-2011 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13
14 14 import __future__
15 15 import abc
16 16 import ast
17 17 import atexit
18 18 import functools
19 19 import os
20 20 import re
21 21 import runpy
22 22 import sys
23 23 import tempfile
24 24 import traceback
25 25 import types
26 26 import subprocess
27 27 import warnings
28 28 from io import open as io_open
29 29
30 30 from pickleshare import PickleShareDB
31 31
32 32 from traitlets.config.configurable import SingletonConfigurable
33 33 from IPython.core import oinspect
34 34 from IPython.core import magic
35 35 from IPython.core import page
36 36 from IPython.core import prefilter
37 37 from IPython.core import shadowns
38 38 from IPython.core import ultratb
39 39 from IPython.core.alias import Alias, AliasManager
40 40 from IPython.core.autocall import ExitAutocall
41 41 from IPython.core.builtin_trap import BuiltinTrap
42 42 from IPython.core.events import EventManager, available_events
43 43 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
44 44 from IPython.core.debugger import Pdb
45 45 from IPython.core.display_trap import DisplayTrap
46 46 from IPython.core.displayhook import DisplayHook
47 47 from IPython.core.displaypub import DisplayPublisher
48 48 from IPython.core.error import InputRejected, UsageError
49 49 from IPython.core.extensions import ExtensionManager
50 50 from IPython.core.formatters import DisplayFormatter
51 51 from IPython.core.history import HistoryManager
52 52 from IPython.core.inputsplitter import ESC_MAGIC, ESC_MAGIC2
53 53 from IPython.core.logger import Logger
54 54 from IPython.core.macro import Macro
55 55 from IPython.core.payload import PayloadManager
56 56 from IPython.core.prefilter import PrefilterManager
57 57 from IPython.core.profiledir import ProfileDir
58 58 from IPython.core.usage import default_banner
59 59 from IPython.testing.skipdoctest import skip_doctest
60 60 from IPython.utils import PyColorize
61 61 from IPython.utils import io
62 62 from IPython.utils import py3compat
63 63 from IPython.utils import openpy
64 64 from IPython.utils.decorators import undoc
65 65 from IPython.utils.io import ask_yes_no
66 66 from IPython.utils.ipstruct import Struct
67 67 from IPython.paths import get_ipython_dir
68 68 from IPython.utils.path import get_home_dir, get_py_filename, ensure_dir_exists
69 69 from IPython.utils.process import system, getoutput
70 70 from IPython.utils.py3compat import builtin_mod, with_metaclass
71 71 from IPython.utils.strdispatch import StrDispatch
72 72 from IPython.utils.syspathcontext import prepended_to_syspath
73 73 from IPython.utils.text import format_screen, LSString, SList, DollarFormatter
74 74 from IPython.utils.tempdir import TemporaryDirectory
75 75 from traitlets import (
76 76 Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type,
77 77 observe, default,
78 78 )
79 79 from warnings import warn
80 80 from logging import error
81 81 import IPython.core.hooks
82 82
83 83 # NoOpContext is deprecated, but ipykernel imports it from here.
84 84 # See https://github.com/ipython/ipykernel/issues/157
85 85 from IPython.utils.contexts import NoOpContext
86 86
87 87 try:
88 88 import docrepr.sphinxify as sphx
89 89
90 90 def sphinxify(doc):
91 91 with TemporaryDirectory() as dirname:
92 92 return {
93 93 'text/html': sphx.sphinxify(doc, dirname),
94 94 'text/plain': doc
95 95 }
96 96 except ImportError:
97 97 sphinxify = None
98 98
99 99
100 100 class ProvisionalWarning(DeprecationWarning):
101 101 """
102 102 Warning class for unstable features
103 103 """
104 104 pass
105 105
106 106 #-----------------------------------------------------------------------------
107 107 # Globals
108 108 #-----------------------------------------------------------------------------
109 109
110 110 # compiled regexps for autoindent management
111 111 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
112 112
113 113 #-----------------------------------------------------------------------------
114 114 # Utilities
115 115 #-----------------------------------------------------------------------------
116 116
117 117 @undoc
118 118 def softspace(file, newvalue):
119 119 """Copied from code.py, to remove the dependency"""
120 120
121 121 oldvalue = 0
122 122 try:
123 123 oldvalue = file.softspace
124 124 except AttributeError:
125 125 pass
126 126 try:
127 127 file.softspace = newvalue
128 128 except (AttributeError, TypeError):
129 129 # "attribute-less object" or "read-only attributes"
130 130 pass
131 131 return oldvalue
132 132
133 133 @undoc
134 134 def no_op(*a, **kw): pass
135 135
136 136
137 137 class SpaceInInput(Exception): pass
138 138
139 139
140 140 def get_default_colors():
141 141 "DEPRECATED"
142 142 warn('get_default_color is Deprecated, and is `Neutral` on all platforms.',
143 143 DeprecationWarning, stacklevel=2)
144 144 return 'Neutral'
145 145
146 146
147 147 class SeparateUnicode(Unicode):
148 148 r"""A Unicode subclass to validate separate_in, separate_out, etc.
149 149
150 150 This is a Unicode based trait that converts '0'->'' and ``'\\n'->'\n'``.
151 151 """
152 152
153 153 def validate(self, obj, value):
154 154 if value == '0': value = ''
155 155 value = value.replace('\\n','\n')
156 156 return super(SeparateUnicode, self).validate(obj, value)
157 157
158 158
159 159 @undoc
160 160 class DummyMod(object):
161 161 """A dummy module used for IPython's interactive module when
162 162 a namespace must be assigned to the module's __dict__."""
163 163 pass
164 164
165 165
166 166 class ExecutionResult(object):
167 167 """The result of a call to :meth:`InteractiveShell.run_cell`
168 168
169 169 Stores information about what took place.
170 170 """
171 171 execution_count = None
172 172 error_before_exec = None
173 173 error_in_exec = None
174 174 result = None
175 175
176 176 @property
177 177 def success(self):
178 178 return (self.error_before_exec is None) and (self.error_in_exec is None)
179 179
180 180 def raise_error(self):
181 181 """Reraises error if `success` is `False`, otherwise does nothing"""
182 182 if self.error_before_exec is not None:
183 183 raise self.error_before_exec
184 184 if self.error_in_exec is not None:
185 185 raise self.error_in_exec
186 186
187 187 def __repr__(self):
188 188 name = self.__class__.__qualname__
189 189 return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s result=%s>' %\
190 190 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.result))
191 191
192 192
193 193 class InteractiveShell(SingletonConfigurable):
194 194 """An enhanced, interactive shell for Python."""
195 195
196 196 _instance = None
197 197
198 198 ast_transformers = List([], help=
199 199 """
200 200 A list of ast.NodeTransformer subclass instances, which will be applied
201 201 to user input before code is run.
202 202 """
203 203 ).tag(config=True)
204 204
205 205 autocall = Enum((0,1,2), default_value=0, help=
206 206 """
207 207 Make IPython automatically call any callable object even if you didn't
208 208 type explicit parentheses. For example, 'str 43' becomes 'str(43)'
209 209 automatically. The value can be '0' to disable the feature, '1' for
210 210 'smart' autocall, where it is not applied if there are no more
211 211 arguments on the line, and '2' for 'full' autocall, where all callable
212 212 objects are automatically called (even if no arguments are present).
213 213 """
214 214 ).tag(config=True)
215 215 # TODO: remove all autoindent logic and put into frontends.
216 216 # We can't do this yet because even runlines uses the autoindent.
217 217 autoindent = Bool(True, help=
218 218 """
219 219 Autoindent IPython code entered interactively.
220 220 """
221 221 ).tag(config=True)
222 222
223 223 automagic = Bool(True, help=
224 224 """
225 225 Enable magic commands to be called without the leading %.
226 226 """
227 227 ).tag(config=True)
228 228
229 229 banner1 = Unicode(default_banner,
230 230 help="""The part of the banner to be printed before the profile"""
231 231 ).tag(config=True)
232 232 banner2 = Unicode('',
233 233 help="""The part of the banner to be printed after the profile"""
234 234 ).tag(config=True)
235 235
236 236 cache_size = Integer(1000, help=
237 237 """
238 238 Set the size of the output cache. The default is 1000, you can
239 239 change it permanently in your config file. Setting it to 0 completely
240 240 disables the caching system, and the minimum value accepted is 20 (if
241 241 you provide a value less than 20, it is reset to 0 and a warning is
242 242 issued). This limit is defined because otherwise you'll spend more
243 243 time re-flushing a too small cache than working
244 244 """
245 245 ).tag(config=True)
246 246 color_info = Bool(True, help=
247 247 """
248 248 Use colors for displaying information about objects. Because this
249 249 information is passed through a pager (like 'less'), and some pagers
250 250 get confused with color codes, this capability can be turned off.
251 251 """
252 252 ).tag(config=True)
253 253 colors = CaselessStrEnum(('Neutral', 'NoColor','LightBG','Linux'),
254 254 default_value='Neutral',
255 255 help="Set the color scheme (NoColor, Neutral, Linux, or LightBG)."
256 256 ).tag(config=True)
257 257 debug = Bool(False).tag(config=True)
258 258 disable_failing_post_execute = Bool(False,
259 259 help="Don't call post-execute functions that have failed in the past."
260 260 ).tag(config=True)
261 261 display_formatter = Instance(DisplayFormatter, allow_none=True)
262 262 displayhook_class = Type(DisplayHook)
263 263 display_pub_class = Type(DisplayPublisher)
264 264
265 265 sphinxify_docstring = Bool(False, help=
266 266 """
267 267 Enables rich html representation of docstrings. (This requires the
268 268 docrepr module).
269 269 """).tag(config=True)
270 270
271 271 @observe("sphinxify_docstring")
272 272 def _sphinxify_docstring_changed(self, change):
273 273 if change['new']:
274 274 warn("`sphinxify_docstring` is provisional since IPython 5.0 and might change in future versions." , ProvisionalWarning)
275 275
276 276 enable_html_pager = Bool(False, help=
277 277 """
278 278 (Provisional API) enables html representation in mime bundles sent
279 279 to pagers.
280 280 """).tag(config=True)
281 281
282 282 @observe("enable_html_pager")
283 283 def _enable_html_pager_changed(self, change):
284 284 if change['new']:
285 285 warn("`enable_html_pager` is provisional since IPython 5.0 and might change in future versions.", ProvisionalWarning)
286 286
287 287 data_pub_class = None
288 288
289 289 exit_now = Bool(False)
290 290 exiter = Instance(ExitAutocall)
291 291 @default('exiter')
292 292 def _exiter_default(self):
293 293 return ExitAutocall(self)
294 294 # Monotonically increasing execution counter
295 295 execution_count = Integer(1)
296 296 filename = Unicode("<ipython console>")
297 297 ipython_dir= Unicode('').tag(config=True) # Set to get_ipython_dir() in __init__
298 298
299 299 # Input splitter, to transform input line by line and detect when a block
300 300 # is ready to be executed.
301 301 input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
302 302 (), {'line_input_checker': True})
303 303
304 304 # This InputSplitter instance is used to transform completed cells before
305 305 # running them. It allows cell magics to contain blank lines.
306 306 input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
307 307 (), {'line_input_checker': False})
308 308
309 309 logstart = Bool(False, help=
310 310 """
311 311 Start logging to the default log file in overwrite mode.
312 312 Use `logappend` to specify a log file to **append** logs to.
313 313 """
314 314 ).tag(config=True)
315 315 logfile = Unicode('', help=
316 316 """
317 317 The name of the logfile to use.
318 318 """
319 319 ).tag(config=True)
320 320 logappend = Unicode('', help=
321 321 """
322 322 Start logging to the given file in append mode.
323 323 Use `logfile` to specify a log file to **overwrite** logs to.
324 324 """
325 325 ).tag(config=True)
326 326 object_info_string_level = Enum((0,1,2), default_value=0,
327 327 ).tag(config=True)
328 328 pdb = Bool(False, help=
329 329 """
330 330 Automatically call the pdb debugger after every exception.
331 331 """
332 332 ).tag(config=True)
333 333 display_page = Bool(False,
334 334 help="""If True, anything that would be passed to the pager
335 335 will be displayed as regular output instead."""
336 336 ).tag(config=True)
337 337
338 338 # deprecated prompt traits:
339 339
340 340 prompt_in1 = Unicode('In [\\#]: ',
341 341 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
342 342 ).tag(config=True)
343 343 prompt_in2 = Unicode(' .\\D.: ',
344 344 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
345 345 ).tag(config=True)
346 346 prompt_out = Unicode('Out[\\#]: ',
347 347 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
348 348 ).tag(config=True)
349 349 prompts_pad_left = Bool(True,
350 350 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
351 351 ).tag(config=True)
352 352
353 353 @observe('prompt_in1', 'prompt_in2', 'prompt_out', 'prompt_pad_left')
354 354 def _prompt_trait_changed(self, change):
355 355 name = change['name']
356 356 warn("InteractiveShell.{name} is deprecated since IPython 4.0"
357 357 " and ignored since 5.0, set TerminalInteractiveShell.prompts"
358 358 " object directly.".format(name=name))
359 359
360 360 # protect against weird cases where self.config may not exist:
361 361
362 362 show_rewritten_input = Bool(True,
363 363 help="Show rewritten input, e.g. for autocall."
364 364 ).tag(config=True)
365 365
366 366 quiet = Bool(False).tag(config=True)
367 367
368 368 history_length = Integer(10000,
369 369 help='Total length of command history'
370 370 ).tag(config=True)
371 371
372 372 history_load_length = Integer(1000, help=
373 373 """
374 374 The number of saved history entries to be loaded
375 375 into the history buffer at startup.
376 376 """
377 377 ).tag(config=True)
378 378
379 379 ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none'],
380 380 default_value='last_expr',
381 381 help="""
382 382 'all', 'last', 'last_expr' or 'none', specifying which nodes should be
383 383 run interactively (displaying output from expressions)."""
384 384 ).tag(config=True)
385 385
386 386 # TODO: this part of prompt management should be moved to the frontends.
387 387 # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
388 388 separate_in = SeparateUnicode('\n').tag(config=True)
389 389 separate_out = SeparateUnicode('').tag(config=True)
390 390 separate_out2 = SeparateUnicode('').tag(config=True)
391 391 wildcards_case_sensitive = Bool(True).tag(config=True)
392 392 xmode = CaselessStrEnum(('Context','Plain', 'Verbose'),
393 393 default_value='Context').tag(config=True)
394 394
395 395 # Subcomponents of InteractiveShell
396 396 alias_manager = Instance('IPython.core.alias.AliasManager', allow_none=True)
397 397 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
398 398 builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap', allow_none=True)
399 399 display_trap = Instance('IPython.core.display_trap.DisplayTrap', allow_none=True)
400 400 extension_manager = Instance('IPython.core.extensions.ExtensionManager', allow_none=True)
401 401 payload_manager = Instance('IPython.core.payload.PayloadManager', allow_none=True)
402 402 history_manager = Instance('IPython.core.history.HistoryAccessorBase', allow_none=True)
403 403 magics_manager = Instance('IPython.core.magic.MagicsManager', allow_none=True)
404 404
405 405 profile_dir = Instance('IPython.core.application.ProfileDir', allow_none=True)
406 406 @property
407 407 def profile(self):
408 408 if self.profile_dir is not None:
409 409 name = os.path.basename(self.profile_dir.location)
410 410 return name.replace('profile_','')
411 411
412 412
413 413 # Private interface
414 414 _post_execute = Dict()
415 415
416 416 # Tracks any GUI loop loaded for pylab
417 417 pylab_gui_select = None
418 418
419 419 last_execution_succeeded = Bool(True, help='Did last executed command succeeded')
420 420
421 421 def __init__(self, ipython_dir=None, profile_dir=None,
422 422 user_module=None, user_ns=None,
423 423 custom_exceptions=((), None), **kwargs):
424 424
425 425 # This is where traits with a config_key argument are updated
426 426 # from the values on config.
427 427 super(InteractiveShell, self).__init__(**kwargs)
428 428 if 'PromptManager' in self.config:
429 429 warn('As of IPython 5.0 `PromptManager` config will have no effect'
430 430 ' and has been replaced by TerminalInteractiveShell.prompts_class')
431 431 self.configurables = [self]
432 432
433 433 # These are relatively independent and stateless
434 434 self.init_ipython_dir(ipython_dir)
435 435 self.init_profile_dir(profile_dir)
436 436 self.init_instance_attrs()
437 437 self.init_environment()
438 438
439 439 # Check if we're in a virtualenv, and set up sys.path.
440 440 self.init_virtualenv()
441 441
442 442 # Create namespaces (user_ns, user_global_ns, etc.)
443 443 self.init_create_namespaces(user_module, user_ns)
444 444 # This has to be done after init_create_namespaces because it uses
445 445 # something in self.user_ns, but before init_sys_modules, which
446 446 # is the first thing to modify sys.
447 447 # TODO: When we override sys.stdout and sys.stderr before this class
448 448 # is created, we are saving the overridden ones here. Not sure if this
449 449 # is what we want to do.
450 450 self.save_sys_module_state()
451 451 self.init_sys_modules()
452 452
453 453 # While we're trying to have each part of the code directly access what
454 454 # it needs without keeping redundant references to objects, we have too
455 455 # much legacy code that expects ip.db to exist.
456 456 self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
457 457
458 458 self.init_history()
459 459 self.init_encoding()
460 460 self.init_prefilter()
461 461
462 462 self.init_syntax_highlighting()
463 463 self.init_hooks()
464 464 self.init_events()
465 465 self.init_pushd_popd_magic()
466 466 self.init_user_ns()
467 467 self.init_logger()
468 468 self.init_builtins()
469 469
470 470 # The following was in post_config_initialization
471 471 self.init_inspector()
472 472 self.raw_input_original = input
473 473 self.init_completer()
474 474 # TODO: init_io() needs to happen before init_traceback handlers
475 475 # because the traceback handlers hardcode the stdout/stderr streams.
476 476 # This logic in in debugger.Pdb and should eventually be changed.
477 477 self.init_io()
478 478 self.init_traceback_handlers(custom_exceptions)
479 479 self.init_prompts()
480 480 self.init_display_formatter()
481 481 self.init_display_pub()
482 482 self.init_data_pub()
483 483 self.init_displayhook()
484 484 self.init_magics()
485 485 self.init_alias()
486 486 self.init_logstart()
487 487 self.init_pdb()
488 488 self.init_extension_manager()
489 489 self.init_payload()
490 490 self.init_deprecation_warnings()
491 491 self.hooks.late_startup_hook()
492 492 self.events.trigger('shell_initialized', self)
493 493 atexit.register(self.atexit_operations)
494 494
495 495 def get_ipython(self):
496 496 """Return the currently running IPython instance."""
497 497 return self
498 498
499 499 #-------------------------------------------------------------------------
500 500 # Trait changed handlers
501 501 #-------------------------------------------------------------------------
502 502 @observe('ipython_dir')
503 503 def _ipython_dir_changed(self, change):
504 504 ensure_dir_exists(change['new'])
505 505
506 506 def set_autoindent(self,value=None):
507 507 """Set the autoindent flag.
508 508
509 509 If called with no arguments, it acts as a toggle."""
510 510 if value is None:
511 511 self.autoindent = not self.autoindent
512 512 else:
513 513 self.autoindent = value
514 514
515 515 #-------------------------------------------------------------------------
516 516 # init_* methods called by __init__
517 517 #-------------------------------------------------------------------------
518 518
519 519 def init_ipython_dir(self, ipython_dir):
520 520 if ipython_dir is not None:
521 521 self.ipython_dir = ipython_dir
522 522 return
523 523
524 524 self.ipython_dir = get_ipython_dir()
525 525
526 526 def init_profile_dir(self, profile_dir):
527 527 if profile_dir is not None:
528 528 self.profile_dir = profile_dir
529 529 return
530 530 self.profile_dir =\
531 531 ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
532 532
533 533 def init_instance_attrs(self):
534 534 self.more = False
535 535
536 536 # command compiler
537 537 self.compile = CachingCompiler()
538 538
539 539 # Make an empty namespace, which extension writers can rely on both
540 540 # existing and NEVER being used by ipython itself. This gives them a
541 541 # convenient location for storing additional information and state
542 542 # their extensions may require, without fear of collisions with other
543 543 # ipython names that may develop later.
544 544 self.meta = Struct()
545 545
546 546 # Temporary files used for various purposes. Deleted at exit.
547 547 self.tempfiles = []
548 548 self.tempdirs = []
549 549
550 550 # keep track of where we started running (mainly for crash post-mortem)
551 551 # This is not being used anywhere currently.
552 self.starting_dir = py3compat.getcwd()
552 self.starting_dir = os.getcwd()
553 553
554 554 # Indentation management
555 555 self.indent_current_nsp = 0
556 556
557 557 # Dict to track post-execution functions that have been registered
558 558 self._post_execute = {}
559 559
560 560 def init_environment(self):
561 561 """Any changes we need to make to the user's environment."""
562 562 pass
563 563
564 564 def init_encoding(self):
565 565 # Get system encoding at startup time. Certain terminals (like Emacs
566 566 # under Win32 have it set to None, and we need to have a known valid
567 567 # encoding to use in the raw_input() method
568 568 try:
569 569 self.stdin_encoding = sys.stdin.encoding or 'ascii'
570 570 except AttributeError:
571 571 self.stdin_encoding = 'ascii'
572 572
573 573
574 574 @observe('colors')
575 575 def init_syntax_highlighting(self, changes=None):
576 576 # Python source parser/formatter for syntax highlighting
577 577 pyformat = PyColorize.Parser(style=self.colors, parent=self).format
578 578 self.pycolorize = lambda src: pyformat(src,'str')
579 579
580 580 def refresh_style(self):
581 581 # No-op here, used in subclass
582 582 pass
583 583
584 584 def init_pushd_popd_magic(self):
585 585 # for pushd/popd management
586 586 self.home_dir = get_home_dir()
587 587
588 588 self.dir_stack = []
589 589
590 590 def init_logger(self):
591 591 self.logger = Logger(self.home_dir, logfname='ipython_log.py',
592 592 logmode='rotate')
593 593
594 594 def init_logstart(self):
595 595 """Initialize logging in case it was requested at the command line.
596 596 """
597 597 if self.logappend:
598 598 self.magic('logstart %s append' % self.logappend)
599 599 elif self.logfile:
600 600 self.magic('logstart %s' % self.logfile)
601 601 elif self.logstart:
602 602 self.magic('logstart')
603 603
604 604 def init_deprecation_warnings(self):
605 605 """
606 606 register default filter for deprecation warning.
607 607
608 608 This will allow deprecation warning of function used interactively to show
609 609 warning to users, and still hide deprecation warning from libraries import.
610 610 """
611 611 warnings.filterwarnings("default", category=DeprecationWarning, module=self.user_ns.get("__name__"))
612 612
613 613 def init_builtins(self):
614 614 # A single, static flag that we set to True. Its presence indicates
615 615 # that an IPython shell has been created, and we make no attempts at
616 616 # removing on exit or representing the existence of more than one
617 617 # IPython at a time.
618 618 builtin_mod.__dict__['__IPYTHON__'] = True
619 619
620 620 self.builtin_trap = BuiltinTrap(shell=self)
621 621
622 622 def init_inspector(self):
623 623 # Object inspector
624 624 self.inspector = oinspect.Inspector(oinspect.InspectColors,
625 625 PyColorize.ANSICodeColors,
626 626 'NoColor',
627 627 self.object_info_string_level)
628 628
629 629 def init_io(self):
630 630 # This will just use sys.stdout and sys.stderr. If you want to
631 631 # override sys.stdout and sys.stderr themselves, you need to do that
632 632 # *before* instantiating this class, because io holds onto
633 633 # references to the underlying streams.
634 634 # io.std* are deprecated, but don't show our own deprecation warnings
635 635 # during initialization of the deprecated API.
636 636 with warnings.catch_warnings():
637 637 warnings.simplefilter('ignore', DeprecationWarning)
638 638 io.stdout = io.IOStream(sys.stdout)
639 639 io.stderr = io.IOStream(sys.stderr)
640 640
641 641 def init_prompts(self):
642 642 # Set system prompts, so that scripts can decide if they are running
643 643 # interactively.
644 644 sys.ps1 = 'In : '
645 645 sys.ps2 = '...: '
646 646 sys.ps3 = 'Out: '
647 647
648 648 def init_display_formatter(self):
649 649 self.display_formatter = DisplayFormatter(parent=self)
650 650 self.configurables.append(self.display_formatter)
651 651
652 652 def init_display_pub(self):
653 653 self.display_pub = self.display_pub_class(parent=self)
654 654 self.configurables.append(self.display_pub)
655 655
656 656 def init_data_pub(self):
657 657 if not self.data_pub_class:
658 658 self.data_pub = None
659 659 return
660 660 self.data_pub = self.data_pub_class(parent=self)
661 661 self.configurables.append(self.data_pub)
662 662
663 663 def init_displayhook(self):
664 664 # Initialize displayhook, set in/out prompts and printing system
665 665 self.displayhook = self.displayhook_class(
666 666 parent=self,
667 667 shell=self,
668 668 cache_size=self.cache_size,
669 669 )
670 670 self.configurables.append(self.displayhook)
671 671 # This is a context manager that installs/revmoes the displayhook at
672 672 # the appropriate time.
673 673 self.display_trap = DisplayTrap(hook=self.displayhook)
674 674
675 675 def init_virtualenv(self):
676 676 """Add a virtualenv to sys.path so the user can import modules from it.
677 677 This isn't perfect: it doesn't use the Python interpreter with which the
678 678 virtualenv was built, and it ignores the --no-site-packages option. A
679 679 warning will appear suggesting the user installs IPython in the
680 680 virtualenv, but for many cases, it probably works well enough.
681 681
682 682 Adapted from code snippets online.
683 683
684 684 http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
685 685 """
686 686 if 'VIRTUAL_ENV' not in os.environ:
687 687 # Not in a virtualenv
688 688 return
689 689
690 690 # venv detection:
691 691 # stdlib venv may symlink sys.executable, so we can't use realpath.
692 692 # but others can symlink *to* the venv Python, so we can't just use sys.executable.
693 693 # So we just check every item in the symlink tree (generally <= 3)
694 694 p = os.path.normcase(sys.executable)
695 695 paths = [p]
696 696 while os.path.islink(p):
697 697 p = os.path.normcase(os.path.join(os.path.dirname(p), os.readlink(p)))
698 698 paths.append(p)
699 699 p_venv = os.path.normcase(os.environ['VIRTUAL_ENV'])
700 700 if any(p.startswith(p_venv) for p in paths):
701 701 # Running properly in the virtualenv, don't need to do anything
702 702 return
703 703
704 704 warn("Attempting to work in a virtualenv. If you encounter problems, please "
705 705 "install IPython inside the virtualenv.")
706 706 if sys.platform == "win32":
707 707 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages')
708 708 else:
709 709 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',
710 710 'python%d.%d' % sys.version_info[:2], 'site-packages')
711 711
712 712 import site
713 713 sys.path.insert(0, virtual_env)
714 714 site.addsitedir(virtual_env)
715 715
716 716 #-------------------------------------------------------------------------
717 717 # Things related to injections into the sys module
718 718 #-------------------------------------------------------------------------
719 719
720 720 def save_sys_module_state(self):
721 721 """Save the state of hooks in the sys module.
722 722
723 723 This has to be called after self.user_module is created.
724 724 """
725 725 self._orig_sys_module_state = {'stdin': sys.stdin,
726 726 'stdout': sys.stdout,
727 727 'stderr': sys.stderr,
728 728 'excepthook': sys.excepthook}
729 729 self._orig_sys_modules_main_name = self.user_module.__name__
730 730 self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
731 731
732 732 def restore_sys_module_state(self):
733 733 """Restore the state of the sys module."""
734 734 try:
735 735 for k, v in self._orig_sys_module_state.items():
736 736 setattr(sys, k, v)
737 737 except AttributeError:
738 738 pass
739 739 # Reset what what done in self.init_sys_modules
740 740 if self._orig_sys_modules_main_mod is not None:
741 741 sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
742 742
743 743 #-------------------------------------------------------------------------
744 744 # Things related to the banner
745 745 #-------------------------------------------------------------------------
746 746
747 747 @property
748 748 def banner(self):
749 749 banner = self.banner1
750 750 if self.profile and self.profile != 'default':
751 751 banner += '\nIPython profile: %s\n' % self.profile
752 752 if self.banner2:
753 753 banner += '\n' + self.banner2
754 754 return banner
755 755
756 756 def show_banner(self, banner=None):
757 757 if banner is None:
758 758 banner = self.banner
759 759 sys.stdout.write(banner)
760 760
761 761 #-------------------------------------------------------------------------
762 762 # Things related to hooks
763 763 #-------------------------------------------------------------------------
764 764
765 765 def init_hooks(self):
766 766 # hooks holds pointers used for user-side customizations
767 767 self.hooks = Struct()
768 768
769 769 self.strdispatchers = {}
770 770
771 771 # Set all default hooks, defined in the IPython.hooks module.
772 772 hooks = IPython.core.hooks
773 773 for hook_name in hooks.__all__:
774 774 # default hooks have priority 100, i.e. low; user hooks should have
775 775 # 0-100 priority
776 776 self.set_hook(hook_name,getattr(hooks,hook_name), 100, _warn_deprecated=False)
777 777
778 778 if self.display_page:
779 779 self.set_hook('show_in_pager', page.as_hook(page.display_page), 90)
780 780
781 781 def set_hook(self,name,hook, priority=50, str_key=None, re_key=None,
782 782 _warn_deprecated=True):
783 783 """set_hook(name,hook) -> sets an internal IPython hook.
784 784
785 785 IPython exposes some of its internal API as user-modifiable hooks. By
786 786 adding your function to one of these hooks, you can modify IPython's
787 787 behavior to call at runtime your own routines."""
788 788
789 789 # At some point in the future, this should validate the hook before it
790 790 # accepts it. Probably at least check that the hook takes the number
791 791 # of args it's supposed to.
792 792
793 793 f = types.MethodType(hook,self)
794 794
795 795 # check if the hook is for strdispatcher first
796 796 if str_key is not None:
797 797 sdp = self.strdispatchers.get(name, StrDispatch())
798 798 sdp.add_s(str_key, f, priority )
799 799 self.strdispatchers[name] = sdp
800 800 return
801 801 if re_key is not None:
802 802 sdp = self.strdispatchers.get(name, StrDispatch())
803 803 sdp.add_re(re.compile(re_key), f, priority )
804 804 self.strdispatchers[name] = sdp
805 805 return
806 806
807 807 dp = getattr(self.hooks, name, None)
808 808 if name not in IPython.core.hooks.__all__:
809 809 print("Warning! Hook '%s' is not one of %s" % \
810 810 (name, IPython.core.hooks.__all__ ))
811 811
812 812 if _warn_deprecated and (name in IPython.core.hooks.deprecated):
813 813 alternative = IPython.core.hooks.deprecated[name]
814 814 warn("Hook {} is deprecated. Use {} instead.".format(name, alternative), stacklevel=2)
815 815
816 816 if not dp:
817 817 dp = IPython.core.hooks.CommandChainDispatcher()
818 818
819 819 try:
820 820 dp.add(f,priority)
821 821 except AttributeError:
822 822 # it was not commandchain, plain old func - replace
823 823 dp = f
824 824
825 825 setattr(self.hooks,name, dp)
826 826
827 827 #-------------------------------------------------------------------------
828 828 # Things related to events
829 829 #-------------------------------------------------------------------------
830 830
831 831 def init_events(self):
832 832 self.events = EventManager(self, available_events)
833 833
834 834 self.events.register("pre_execute", self._clear_warning_registry)
835 835
836 836 def register_post_execute(self, func):
837 837 """DEPRECATED: Use ip.events.register('post_run_cell', func)
838 838
839 839 Register a function for calling after code execution.
840 840 """
841 841 warn("ip.register_post_execute is deprecated, use "
842 842 "ip.events.register('post_run_cell', func) instead.", stacklevel=2)
843 843 self.events.register('post_run_cell', func)
844 844
845 845 def _clear_warning_registry(self):
846 846 # clear the warning registry, so that different code blocks with
847 847 # overlapping line number ranges don't cause spurious suppression of
848 848 # warnings (see gh-6611 for details)
849 849 if "__warningregistry__" in self.user_global_ns:
850 850 del self.user_global_ns["__warningregistry__"]
851 851
852 852 #-------------------------------------------------------------------------
853 853 # Things related to the "main" module
854 854 #-------------------------------------------------------------------------
855 855
856 856 def new_main_mod(self, filename, modname):
857 857 """Return a new 'main' module object for user code execution.
858 858
859 859 ``filename`` should be the path of the script which will be run in the
860 860 module. Requests with the same filename will get the same module, with
861 861 its namespace cleared.
862 862
863 863 ``modname`` should be the module name - normally either '__main__' or
864 864 the basename of the file without the extension.
865 865
866 866 When scripts are executed via %run, we must keep a reference to their
867 867 __main__ module around so that Python doesn't
868 868 clear it, rendering references to module globals useless.
869 869
870 870 This method keeps said reference in a private dict, keyed by the
871 871 absolute path of the script. This way, for multiple executions of the
872 872 same script we only keep one copy of the namespace (the last one),
873 873 thus preventing memory leaks from old references while allowing the
874 874 objects from the last execution to be accessible.
875 875 """
876 876 filename = os.path.abspath(filename)
877 877 try:
878 878 main_mod = self._main_mod_cache[filename]
879 879 except KeyError:
880 880 main_mod = self._main_mod_cache[filename] = types.ModuleType(
881 881 py3compat.cast_bytes_py2(modname),
882 882 doc="Module created for script run in IPython")
883 883 else:
884 884 main_mod.__dict__.clear()
885 885 main_mod.__name__ = modname
886 886
887 887 main_mod.__file__ = filename
888 888 # It seems pydoc (and perhaps others) needs any module instance to
889 889 # implement a __nonzero__ method
890 890 main_mod.__nonzero__ = lambda : True
891 891
892 892 return main_mod
893 893
894 894 def clear_main_mod_cache(self):
895 895 """Clear the cache of main modules.
896 896
897 897 Mainly for use by utilities like %reset.
898 898
899 899 Examples
900 900 --------
901 901
902 902 In [15]: import IPython
903 903
904 904 In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
905 905
906 906 In [17]: len(_ip._main_mod_cache) > 0
907 907 Out[17]: True
908 908
909 909 In [18]: _ip.clear_main_mod_cache()
910 910
911 911 In [19]: len(_ip._main_mod_cache) == 0
912 912 Out[19]: True
913 913 """
914 914 self._main_mod_cache.clear()
915 915
916 916 #-------------------------------------------------------------------------
917 917 # Things related to debugging
918 918 #-------------------------------------------------------------------------
919 919
920 920 def init_pdb(self):
921 921 # Set calling of pdb on exceptions
922 922 # self.call_pdb is a property
923 923 self.call_pdb = self.pdb
924 924
925 925 def _get_call_pdb(self):
926 926 return self._call_pdb
927 927
928 928 def _set_call_pdb(self,val):
929 929
930 930 if val not in (0,1,False,True):
931 931 raise ValueError('new call_pdb value must be boolean')
932 932
933 933 # store value in instance
934 934 self._call_pdb = val
935 935
936 936 # notify the actual exception handlers
937 937 self.InteractiveTB.call_pdb = val
938 938
939 939 call_pdb = property(_get_call_pdb,_set_call_pdb,None,
940 940 'Control auto-activation of pdb at exceptions')
941 941
942 942 def debugger(self,force=False):
943 943 """Call the pdb debugger.
944 944
945 945 Keywords:
946 946
947 947 - force(False): by default, this routine checks the instance call_pdb
948 948 flag and does not actually invoke the debugger if the flag is false.
949 949 The 'force' option forces the debugger to activate even if the flag
950 950 is false.
951 951 """
952 952
953 953 if not (force or self.call_pdb):
954 954 return
955 955
956 956 if not hasattr(sys,'last_traceback'):
957 957 error('No traceback has been produced, nothing to debug.')
958 958 return
959 959
960 960 self.InteractiveTB.debugger(force=True)
961 961
962 962 #-------------------------------------------------------------------------
963 963 # Things related to IPython's various namespaces
964 964 #-------------------------------------------------------------------------
965 965 default_user_namespaces = True
966 966
967 967 def init_create_namespaces(self, user_module=None, user_ns=None):
968 968 # Create the namespace where the user will operate. user_ns is
969 969 # normally the only one used, and it is passed to the exec calls as
970 970 # the locals argument. But we do carry a user_global_ns namespace
971 971 # given as the exec 'globals' argument, This is useful in embedding
972 972 # situations where the ipython shell opens in a context where the
973 973 # distinction between locals and globals is meaningful. For
974 974 # non-embedded contexts, it is just the same object as the user_ns dict.
975 975
976 976 # FIXME. For some strange reason, __builtins__ is showing up at user
977 977 # level as a dict instead of a module. This is a manual fix, but I
978 978 # should really track down where the problem is coming from. Alex
979 979 # Schmolck reported this problem first.
980 980
981 981 # A useful post by Alex Martelli on this topic:
982 982 # Re: inconsistent value from __builtins__
983 983 # Von: Alex Martelli <aleaxit@yahoo.com>
984 984 # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
985 985 # Gruppen: comp.lang.python
986 986
987 987 # Michael Hohn <hohn@hooknose.lbl.gov> wrote:
988 988 # > >>> print type(builtin_check.get_global_binding('__builtins__'))
989 989 # > <type 'dict'>
990 990 # > >>> print type(__builtins__)
991 991 # > <type 'module'>
992 992 # > Is this difference in return value intentional?
993 993
994 994 # Well, it's documented that '__builtins__' can be either a dictionary
995 995 # or a module, and it's been that way for a long time. Whether it's
996 996 # intentional (or sensible), I don't know. In any case, the idea is
997 997 # that if you need to access the built-in namespace directly, you
998 998 # should start with "import __builtin__" (note, no 's') which will
999 999 # definitely give you a module. Yeah, it's somewhat confusing:-(.
1000 1000
1001 1001 # These routines return a properly built module and dict as needed by
1002 1002 # the rest of the code, and can also be used by extension writers to
1003 1003 # generate properly initialized namespaces.
1004 1004 if (user_ns is not None) or (user_module is not None):
1005 1005 self.default_user_namespaces = False
1006 1006 self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
1007 1007
1008 1008 # A record of hidden variables we have added to the user namespace, so
1009 1009 # we can list later only variables defined in actual interactive use.
1010 1010 self.user_ns_hidden = {}
1011 1011
1012 1012 # Now that FakeModule produces a real module, we've run into a nasty
1013 1013 # problem: after script execution (via %run), the module where the user
1014 1014 # code ran is deleted. Now that this object is a true module (needed
1015 1015 # so doctest and other tools work correctly), the Python module
1016 1016 # teardown mechanism runs over it, and sets to None every variable
1017 1017 # present in that module. Top-level references to objects from the
1018 1018 # script survive, because the user_ns is updated with them. However,
1019 1019 # calling functions defined in the script that use other things from
1020 1020 # the script will fail, because the function's closure had references
1021 1021 # to the original objects, which are now all None. So we must protect
1022 1022 # these modules from deletion by keeping a cache.
1023 1023 #
1024 1024 # To avoid keeping stale modules around (we only need the one from the
1025 1025 # last run), we use a dict keyed with the full path to the script, so
1026 1026 # only the last version of the module is held in the cache. Note,
1027 1027 # however, that we must cache the module *namespace contents* (their
1028 1028 # __dict__). Because if we try to cache the actual modules, old ones
1029 1029 # (uncached) could be destroyed while still holding references (such as
1030 1030 # those held by GUI objects that tend to be long-lived)>
1031 1031 #
1032 1032 # The %reset command will flush this cache. See the cache_main_mod()
1033 1033 # and clear_main_mod_cache() methods for details on use.
1034 1034
1035 1035 # This is the cache used for 'main' namespaces
1036 1036 self._main_mod_cache = {}
1037 1037
1038 1038 # A table holding all the namespaces IPython deals with, so that
1039 1039 # introspection facilities can search easily.
1040 1040 self.ns_table = {'user_global':self.user_module.__dict__,
1041 1041 'user_local':self.user_ns,
1042 1042 'builtin':builtin_mod.__dict__
1043 1043 }
1044 1044
1045 1045 @property
1046 1046 def user_global_ns(self):
1047 1047 return self.user_module.__dict__
1048 1048
1049 1049 def prepare_user_module(self, user_module=None, user_ns=None):
1050 1050 """Prepare the module and namespace in which user code will be run.
1051 1051
1052 1052 When IPython is started normally, both parameters are None: a new module
1053 1053 is created automatically, and its __dict__ used as the namespace.
1054 1054
1055 1055 If only user_module is provided, its __dict__ is used as the namespace.
1056 1056 If only user_ns is provided, a dummy module is created, and user_ns
1057 1057 becomes the global namespace. If both are provided (as they may be
1058 1058 when embedding), user_ns is the local namespace, and user_module
1059 1059 provides the global namespace.
1060 1060
1061 1061 Parameters
1062 1062 ----------
1063 1063 user_module : module, optional
1064 1064 The current user module in which IPython is being run. If None,
1065 1065 a clean module will be created.
1066 1066 user_ns : dict, optional
1067 1067 A namespace in which to run interactive commands.
1068 1068
1069 1069 Returns
1070 1070 -------
1071 1071 A tuple of user_module and user_ns, each properly initialised.
1072 1072 """
1073 1073 if user_module is None and user_ns is not None:
1074 1074 user_ns.setdefault("__name__", "__main__")
1075 1075 user_module = DummyMod()
1076 1076 user_module.__dict__ = user_ns
1077 1077
1078 1078 if user_module is None:
1079 1079 user_module = types.ModuleType("__main__",
1080 1080 doc="Automatically created module for IPython interactive environment")
1081 1081
1082 1082 # We must ensure that __builtin__ (without the final 's') is always
1083 1083 # available and pointing to the __builtin__ *module*. For more details:
1084 1084 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1085 1085 user_module.__dict__.setdefault('__builtin__', builtin_mod)
1086 1086 user_module.__dict__.setdefault('__builtins__', builtin_mod)
1087 1087
1088 1088 if user_ns is None:
1089 1089 user_ns = user_module.__dict__
1090 1090
1091 1091 return user_module, user_ns
1092 1092
1093 1093 def init_sys_modules(self):
1094 1094 # We need to insert into sys.modules something that looks like a
1095 1095 # module but which accesses the IPython namespace, for shelve and
1096 1096 # pickle to work interactively. Normally they rely on getting
1097 1097 # everything out of __main__, but for embedding purposes each IPython
1098 1098 # instance has its own private namespace, so we can't go shoving
1099 1099 # everything into __main__.
1100 1100
1101 1101 # note, however, that we should only do this for non-embedded
1102 1102 # ipythons, which really mimic the __main__.__dict__ with their own
1103 1103 # namespace. Embedded instances, on the other hand, should not do
1104 1104 # this because they need to manage the user local/global namespaces
1105 1105 # only, but they live within a 'normal' __main__ (meaning, they
1106 1106 # shouldn't overtake the execution environment of the script they're
1107 1107 # embedded in).
1108 1108
1109 1109 # This is overridden in the InteractiveShellEmbed subclass to a no-op.
1110 1110 main_name = self.user_module.__name__
1111 1111 sys.modules[main_name] = self.user_module
1112 1112
1113 1113 def init_user_ns(self):
1114 1114 """Initialize all user-visible namespaces to their minimum defaults.
1115 1115
1116 1116 Certain history lists are also initialized here, as they effectively
1117 1117 act as user namespaces.
1118 1118
1119 1119 Notes
1120 1120 -----
1121 1121 All data structures here are only filled in, they are NOT reset by this
1122 1122 method. If they were not empty before, data will simply be added to
1123 1123 therm.
1124 1124 """
1125 1125 # This function works in two parts: first we put a few things in
1126 1126 # user_ns, and we sync that contents into user_ns_hidden so that these
1127 1127 # initial variables aren't shown by %who. After the sync, we add the
1128 1128 # rest of what we *do* want the user to see with %who even on a new
1129 1129 # session (probably nothing, so they really only see their own stuff)
1130 1130
1131 1131 # The user dict must *always* have a __builtin__ reference to the
1132 1132 # Python standard __builtin__ namespace, which must be imported.
1133 1133 # This is so that certain operations in prompt evaluation can be
1134 1134 # reliably executed with builtins. Note that we can NOT use
1135 1135 # __builtins__ (note the 's'), because that can either be a dict or a
1136 1136 # module, and can even mutate at runtime, depending on the context
1137 1137 # (Python makes no guarantees on it). In contrast, __builtin__ is
1138 1138 # always a module object, though it must be explicitly imported.
1139 1139
1140 1140 # For more details:
1141 1141 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1142 1142 ns = dict()
1143 1143
1144 1144 # make global variables for user access to the histories
1145 1145 ns['_ih'] = self.history_manager.input_hist_parsed
1146 1146 ns['_oh'] = self.history_manager.output_hist
1147 1147 ns['_dh'] = self.history_manager.dir_hist
1148 1148
1149 1149 ns['_sh'] = shadowns
1150 1150
1151 1151 # user aliases to input and output histories. These shouldn't show up
1152 1152 # in %who, as they can have very large reprs.
1153 1153 ns['In'] = self.history_manager.input_hist_parsed
1154 1154 ns['Out'] = self.history_manager.output_hist
1155 1155
1156 1156 # Store myself as the public api!!!
1157 1157 ns['get_ipython'] = self.get_ipython
1158 1158
1159 1159 ns['exit'] = self.exiter
1160 1160 ns['quit'] = self.exiter
1161 1161
1162 1162 # Sync what we've added so far to user_ns_hidden so these aren't seen
1163 1163 # by %who
1164 1164 self.user_ns_hidden.update(ns)
1165 1165
1166 1166 # Anything put into ns now would show up in %who. Think twice before
1167 1167 # putting anything here, as we really want %who to show the user their
1168 1168 # stuff, not our variables.
1169 1169
1170 1170 # Finally, update the real user's namespace
1171 1171 self.user_ns.update(ns)
1172 1172
1173 1173 @property
1174 1174 def all_ns_refs(self):
1175 1175 """Get a list of references to all the namespace dictionaries in which
1176 1176 IPython might store a user-created object.
1177 1177
1178 1178 Note that this does not include the displayhook, which also caches
1179 1179 objects from the output."""
1180 1180 return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
1181 1181 [m.__dict__ for m in self._main_mod_cache.values()]
1182 1182
1183 1183 def reset(self, new_session=True):
1184 1184 """Clear all internal namespaces, and attempt to release references to
1185 1185 user objects.
1186 1186
1187 1187 If new_session is True, a new history session will be opened.
1188 1188 """
1189 1189 # Clear histories
1190 1190 self.history_manager.reset(new_session)
1191 1191 # Reset counter used to index all histories
1192 1192 if new_session:
1193 1193 self.execution_count = 1
1194 1194
1195 1195 # Flush cached output items
1196 1196 if self.displayhook.do_full_cache:
1197 1197 self.displayhook.flush()
1198 1198
1199 1199 # The main execution namespaces must be cleared very carefully,
1200 1200 # skipping the deletion of the builtin-related keys, because doing so
1201 1201 # would cause errors in many object's __del__ methods.
1202 1202 if self.user_ns is not self.user_global_ns:
1203 1203 self.user_ns.clear()
1204 1204 ns = self.user_global_ns
1205 1205 drop_keys = set(ns.keys())
1206 1206 drop_keys.discard('__builtin__')
1207 1207 drop_keys.discard('__builtins__')
1208 1208 drop_keys.discard('__name__')
1209 1209 for k in drop_keys:
1210 1210 del ns[k]
1211 1211
1212 1212 self.user_ns_hidden.clear()
1213 1213
1214 1214 # Restore the user namespaces to minimal usability
1215 1215 self.init_user_ns()
1216 1216
1217 1217 # Restore the default and user aliases
1218 1218 self.alias_manager.clear_aliases()
1219 1219 self.alias_manager.init_aliases()
1220 1220
1221 1221 # Flush the private list of module references kept for script
1222 1222 # execution protection
1223 1223 self.clear_main_mod_cache()
1224 1224
1225 1225 def del_var(self, varname, by_name=False):
1226 1226 """Delete a variable from the various namespaces, so that, as
1227 1227 far as possible, we're not keeping any hidden references to it.
1228 1228
1229 1229 Parameters
1230 1230 ----------
1231 1231 varname : str
1232 1232 The name of the variable to delete.
1233 1233 by_name : bool
1234 1234 If True, delete variables with the given name in each
1235 1235 namespace. If False (default), find the variable in the user
1236 1236 namespace, and delete references to it.
1237 1237 """
1238 1238 if varname in ('__builtin__', '__builtins__'):
1239 1239 raise ValueError("Refusing to delete %s" % varname)
1240 1240
1241 1241 ns_refs = self.all_ns_refs
1242 1242
1243 1243 if by_name: # Delete by name
1244 1244 for ns in ns_refs:
1245 1245 try:
1246 1246 del ns[varname]
1247 1247 except KeyError:
1248 1248 pass
1249 1249 else: # Delete by object
1250 1250 try:
1251 1251 obj = self.user_ns[varname]
1252 1252 except KeyError:
1253 1253 raise NameError("name '%s' is not defined" % varname)
1254 1254 # Also check in output history
1255 1255 ns_refs.append(self.history_manager.output_hist)
1256 1256 for ns in ns_refs:
1257 1257 to_delete = [n for n, o in ns.items() if o is obj]
1258 1258 for name in to_delete:
1259 1259 del ns[name]
1260 1260
1261 1261 # displayhook keeps extra references, but not in a dictionary
1262 1262 for name in ('_', '__', '___'):
1263 1263 if getattr(self.displayhook, name) is obj:
1264 1264 setattr(self.displayhook, name, None)
1265 1265
1266 1266 def reset_selective(self, regex=None):
1267 1267 """Clear selective variables from internal namespaces based on a
1268 1268 specified regular expression.
1269 1269
1270 1270 Parameters
1271 1271 ----------
1272 1272 regex : string or compiled pattern, optional
1273 1273 A regular expression pattern that will be used in searching
1274 1274 variable names in the users namespaces.
1275 1275 """
1276 1276 if regex is not None:
1277 1277 try:
1278 1278 m = re.compile(regex)
1279 1279 except TypeError:
1280 1280 raise TypeError('regex must be a string or compiled pattern')
1281 1281 # Search for keys in each namespace that match the given regex
1282 1282 # If a match is found, delete the key/value pair.
1283 1283 for ns in self.all_ns_refs:
1284 1284 for var in ns:
1285 1285 if m.search(var):
1286 1286 del ns[var]
1287 1287
1288 1288 def push(self, variables, interactive=True):
1289 1289 """Inject a group of variables into the IPython user namespace.
1290 1290
1291 1291 Parameters
1292 1292 ----------
1293 1293 variables : dict, str or list/tuple of str
1294 1294 The variables to inject into the user's namespace. If a dict, a
1295 1295 simple update is done. If a str, the string is assumed to have
1296 1296 variable names separated by spaces. A list/tuple of str can also
1297 1297 be used to give the variable names. If just the variable names are
1298 1298 give (list/tuple/str) then the variable values looked up in the
1299 1299 callers frame.
1300 1300 interactive : bool
1301 1301 If True (default), the variables will be listed with the ``who``
1302 1302 magic.
1303 1303 """
1304 1304 vdict = None
1305 1305
1306 1306 # We need a dict of name/value pairs to do namespace updates.
1307 1307 if isinstance(variables, dict):
1308 1308 vdict = variables
1309 1309 elif isinstance(variables, (str, list, tuple)):
1310 1310 if isinstance(variables, str):
1311 1311 vlist = variables.split()
1312 1312 else:
1313 1313 vlist = variables
1314 1314 vdict = {}
1315 1315 cf = sys._getframe(1)
1316 1316 for name in vlist:
1317 1317 try:
1318 1318 vdict[name] = eval(name, cf.f_globals, cf.f_locals)
1319 1319 except:
1320 1320 print('Could not get variable %s from %s' %
1321 1321 (name,cf.f_code.co_name))
1322 1322 else:
1323 1323 raise ValueError('variables must be a dict/str/list/tuple')
1324 1324
1325 1325 # Propagate variables to user namespace
1326 1326 self.user_ns.update(vdict)
1327 1327
1328 1328 # And configure interactive visibility
1329 1329 user_ns_hidden = self.user_ns_hidden
1330 1330 if interactive:
1331 1331 for name in vdict:
1332 1332 user_ns_hidden.pop(name, None)
1333 1333 else:
1334 1334 user_ns_hidden.update(vdict)
1335 1335
1336 1336 def drop_by_id(self, variables):
1337 1337 """Remove a dict of variables from the user namespace, if they are the
1338 1338 same as the values in the dictionary.
1339 1339
1340 1340 This is intended for use by extensions: variables that they've added can
1341 1341 be taken back out if they are unloaded, without removing any that the
1342 1342 user has overwritten.
1343 1343
1344 1344 Parameters
1345 1345 ----------
1346 1346 variables : dict
1347 1347 A dictionary mapping object names (as strings) to the objects.
1348 1348 """
1349 1349 for name, obj in variables.items():
1350 1350 if name in self.user_ns and self.user_ns[name] is obj:
1351 1351 del self.user_ns[name]
1352 1352 self.user_ns_hidden.pop(name, None)
1353 1353
1354 1354 #-------------------------------------------------------------------------
1355 1355 # Things related to object introspection
1356 1356 #-------------------------------------------------------------------------
1357 1357
1358 1358 def _ofind(self, oname, namespaces=None):
1359 1359 """Find an object in the available namespaces.
1360 1360
1361 1361 self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
1362 1362
1363 1363 Has special code to detect magic functions.
1364 1364 """
1365 1365 oname = oname.strip()
1366 1366 #print '1- oname: <%r>' % oname # dbg
1367 1367 if not oname.startswith(ESC_MAGIC) and \
1368 1368 not oname.startswith(ESC_MAGIC2) and \
1369 1369 not py3compat.isidentifier(oname, dotted=True):
1370 1370 return dict(found=False)
1371 1371
1372 1372 if namespaces is None:
1373 1373 # Namespaces to search in:
1374 1374 # Put them in a list. The order is important so that we
1375 1375 # find things in the same order that Python finds them.
1376 1376 namespaces = [ ('Interactive', self.user_ns),
1377 1377 ('Interactive (global)', self.user_global_ns),
1378 1378 ('Python builtin', builtin_mod.__dict__),
1379 1379 ]
1380 1380
1381 1381 # initialize results to 'null'
1382 1382 found = False; obj = None; ospace = None;
1383 1383 ismagic = False; isalias = False; parent = None
1384 1384
1385 1385 # We need to special-case 'print', which as of python2.6 registers as a
1386 1386 # function but should only be treated as one if print_function was
1387 1387 # loaded with a future import. In this case, just bail.
1388 1388 if (oname == 'print' and not py3compat.PY3 and not \
1389 1389 (self.compile.compiler_flags & __future__.CO_FUTURE_PRINT_FUNCTION)):
1390 1390 return {'found':found, 'obj':obj, 'namespace':ospace,
1391 1391 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1392 1392
1393 1393 # Look for the given name by splitting it in parts. If the head is
1394 1394 # found, then we look for all the remaining parts as members, and only
1395 1395 # declare success if we can find them all.
1396 1396 oname_parts = oname.split('.')
1397 1397 oname_head, oname_rest = oname_parts[0],oname_parts[1:]
1398 1398 for nsname,ns in namespaces:
1399 1399 try:
1400 1400 obj = ns[oname_head]
1401 1401 except KeyError:
1402 1402 continue
1403 1403 else:
1404 1404 #print 'oname_rest:', oname_rest # dbg
1405 1405 for idx, part in enumerate(oname_rest):
1406 1406 try:
1407 1407 parent = obj
1408 1408 # The last part is looked up in a special way to avoid
1409 1409 # descriptor invocation as it may raise or have side
1410 1410 # effects.
1411 1411 if idx == len(oname_rest) - 1:
1412 1412 obj = self._getattr_property(obj, part)
1413 1413 else:
1414 1414 obj = getattr(obj, part)
1415 1415 except:
1416 1416 # Blanket except b/c some badly implemented objects
1417 1417 # allow __getattr__ to raise exceptions other than
1418 1418 # AttributeError, which then crashes IPython.
1419 1419 break
1420 1420 else:
1421 1421 # If we finish the for loop (no break), we got all members
1422 1422 found = True
1423 1423 ospace = nsname
1424 1424 break # namespace loop
1425 1425
1426 1426 # Try to see if it's magic
1427 1427 if not found:
1428 1428 obj = None
1429 1429 if oname.startswith(ESC_MAGIC2):
1430 1430 oname = oname.lstrip(ESC_MAGIC2)
1431 1431 obj = self.find_cell_magic(oname)
1432 1432 elif oname.startswith(ESC_MAGIC):
1433 1433 oname = oname.lstrip(ESC_MAGIC)
1434 1434 obj = self.find_line_magic(oname)
1435 1435 else:
1436 1436 # search without prefix, so run? will find %run?
1437 1437 obj = self.find_line_magic(oname)
1438 1438 if obj is None:
1439 1439 obj = self.find_cell_magic(oname)
1440 1440 if obj is not None:
1441 1441 found = True
1442 1442 ospace = 'IPython internal'
1443 1443 ismagic = True
1444 1444 isalias = isinstance(obj, Alias)
1445 1445
1446 1446 # Last try: special-case some literals like '', [], {}, etc:
1447 1447 if not found and oname_head in ["''",'""','[]','{}','()']:
1448 1448 obj = eval(oname_head)
1449 1449 found = True
1450 1450 ospace = 'Interactive'
1451 1451
1452 1452 return {'found':found, 'obj':obj, 'namespace':ospace,
1453 1453 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1454 1454
1455 1455 @staticmethod
1456 1456 def _getattr_property(obj, attrname):
1457 1457 """Property-aware getattr to use in object finding.
1458 1458
1459 1459 If attrname represents a property, return it unevaluated (in case it has
1460 1460 side effects or raises an error.
1461 1461
1462 1462 """
1463 1463 if not isinstance(obj, type):
1464 1464 try:
1465 1465 # `getattr(type(obj), attrname)` is not guaranteed to return
1466 1466 # `obj`, but does so for property:
1467 1467 #
1468 1468 # property.__get__(self, None, cls) -> self
1469 1469 #
1470 1470 # The universal alternative is to traverse the mro manually
1471 1471 # searching for attrname in class dicts.
1472 1472 attr = getattr(type(obj), attrname)
1473 1473 except AttributeError:
1474 1474 pass
1475 1475 else:
1476 1476 # This relies on the fact that data descriptors (with both
1477 1477 # __get__ & __set__ magic methods) take precedence over
1478 1478 # instance-level attributes:
1479 1479 #
1480 1480 # class A(object):
1481 1481 # @property
1482 1482 # def foobar(self): return 123
1483 1483 # a = A()
1484 1484 # a.__dict__['foobar'] = 345
1485 1485 # a.foobar # == 123
1486 1486 #
1487 1487 # So, a property may be returned right away.
1488 1488 if isinstance(attr, property):
1489 1489 return attr
1490 1490
1491 1491 # Nothing helped, fall back.
1492 1492 return getattr(obj, attrname)
1493 1493
1494 1494 def _object_find(self, oname, namespaces=None):
1495 1495 """Find an object and return a struct with info about it."""
1496 1496 return Struct(self._ofind(oname, namespaces))
1497 1497
1498 1498 def _inspect(self, meth, oname, namespaces=None, **kw):
1499 1499 """Generic interface to the inspector system.
1500 1500
1501 1501 This function is meant to be called by pdef, pdoc & friends.
1502 1502 """
1503 1503 info = self._object_find(oname, namespaces)
1504 1504 docformat = sphinxify if self.sphinxify_docstring else None
1505 1505 if info.found:
1506 1506 pmethod = getattr(self.inspector, meth)
1507 1507 # TODO: only apply format_screen to the plain/text repr of the mime
1508 1508 # bundle.
1509 1509 formatter = format_screen if info.ismagic else docformat
1510 1510 if meth == 'pdoc':
1511 1511 pmethod(info.obj, oname, formatter)
1512 1512 elif meth == 'pinfo':
1513 1513 pmethod(info.obj, oname, formatter, info,
1514 1514 enable_html_pager=self.enable_html_pager, **kw)
1515 1515 else:
1516 1516 pmethod(info.obj, oname)
1517 1517 else:
1518 1518 print('Object `%s` not found.' % oname)
1519 1519 return 'not found' # so callers can take other action
1520 1520
1521 1521 def object_inspect(self, oname, detail_level=0):
1522 1522 """Get object info about oname"""
1523 1523 with self.builtin_trap:
1524 1524 info = self._object_find(oname)
1525 1525 if info.found:
1526 1526 return self.inspector.info(info.obj, oname, info=info,
1527 1527 detail_level=detail_level
1528 1528 )
1529 1529 else:
1530 1530 return oinspect.object_info(name=oname, found=False)
1531 1531
1532 1532 def object_inspect_text(self, oname, detail_level=0):
1533 1533 """Get object info as formatted text"""
1534 1534 return self.object_inspect_mime(oname, detail_level)['text/plain']
1535 1535
1536 1536 def object_inspect_mime(self, oname, detail_level=0):
1537 1537 """Get object info as a mimebundle of formatted representations.
1538 1538
1539 1539 A mimebundle is a dictionary, keyed by mime-type.
1540 1540 It must always have the key `'text/plain'`.
1541 1541 """
1542 1542 with self.builtin_trap:
1543 1543 info = self._object_find(oname)
1544 1544 if info.found:
1545 1545 return self.inspector._get_info(info.obj, oname, info=info,
1546 1546 detail_level=detail_level
1547 1547 )
1548 1548 else:
1549 1549 raise KeyError(oname)
1550 1550
1551 1551 #-------------------------------------------------------------------------
1552 1552 # Things related to history management
1553 1553 #-------------------------------------------------------------------------
1554 1554
1555 1555 def init_history(self):
1556 1556 """Sets up the command history, and starts regular autosaves."""
1557 1557 self.history_manager = HistoryManager(shell=self, parent=self)
1558 1558 self.configurables.append(self.history_manager)
1559 1559
1560 1560 #-------------------------------------------------------------------------
1561 1561 # Things related to exception handling and tracebacks (not debugging)
1562 1562 #-------------------------------------------------------------------------
1563 1563
1564 1564 debugger_cls = Pdb
1565 1565
1566 1566 def init_traceback_handlers(self, custom_exceptions):
1567 1567 # Syntax error handler.
1568 1568 self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor', parent=self)
1569 1569
1570 1570 # The interactive one is initialized with an offset, meaning we always
1571 1571 # want to remove the topmost item in the traceback, which is our own
1572 1572 # internal code. Valid modes: ['Plain','Context','Verbose']
1573 1573 self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
1574 1574 color_scheme='NoColor',
1575 1575 tb_offset = 1,
1576 1576 check_cache=check_linecache_ipython,
1577 1577 debugger_cls=self.debugger_cls, parent=self)
1578 1578
1579 1579 # The instance will store a pointer to the system-wide exception hook,
1580 1580 # so that runtime code (such as magics) can access it. This is because
1581 1581 # during the read-eval loop, it may get temporarily overwritten.
1582 1582 self.sys_excepthook = sys.excepthook
1583 1583
1584 1584 # and add any custom exception handlers the user may have specified
1585 1585 self.set_custom_exc(*custom_exceptions)
1586 1586
1587 1587 # Set the exception mode
1588 1588 self.InteractiveTB.set_mode(mode=self.xmode)
1589 1589
1590 1590 def set_custom_exc(self, exc_tuple, handler):
1591 1591 """set_custom_exc(exc_tuple, handler)
1592 1592
1593 1593 Set a custom exception handler, which will be called if any of the
1594 1594 exceptions in exc_tuple occur in the mainloop (specifically, in the
1595 1595 run_code() method).
1596 1596
1597 1597 Parameters
1598 1598 ----------
1599 1599
1600 1600 exc_tuple : tuple of exception classes
1601 1601 A *tuple* of exception classes, for which to call the defined
1602 1602 handler. It is very important that you use a tuple, and NOT A
1603 1603 LIST here, because of the way Python's except statement works. If
1604 1604 you only want to trap a single exception, use a singleton tuple::
1605 1605
1606 1606 exc_tuple == (MyCustomException,)
1607 1607
1608 1608 handler : callable
1609 1609 handler must have the following signature::
1610 1610
1611 1611 def my_handler(self, etype, value, tb, tb_offset=None):
1612 1612 ...
1613 1613 return structured_traceback
1614 1614
1615 1615 Your handler must return a structured traceback (a list of strings),
1616 1616 or None.
1617 1617
1618 1618 This will be made into an instance method (via types.MethodType)
1619 1619 of IPython itself, and it will be called if any of the exceptions
1620 1620 listed in the exc_tuple are caught. If the handler is None, an
1621 1621 internal basic one is used, which just prints basic info.
1622 1622
1623 1623 To protect IPython from crashes, if your handler ever raises an
1624 1624 exception or returns an invalid result, it will be immediately
1625 1625 disabled.
1626 1626
1627 1627 WARNING: by putting in your own exception handler into IPython's main
1628 1628 execution loop, you run a very good chance of nasty crashes. This
1629 1629 facility should only be used if you really know what you are doing."""
1630 1630
1631 1631 assert type(exc_tuple)==type(()) , \
1632 1632 "The custom exceptions must be given AS A TUPLE."
1633 1633
1634 1634 def dummy_handler(self, etype, value, tb, tb_offset=None):
1635 1635 print('*** Simple custom exception handler ***')
1636 1636 print('Exception type :',etype)
1637 1637 print('Exception value:',value)
1638 1638 print('Traceback :',tb)
1639 1639 #print 'Source code :','\n'.join(self.buffer)
1640 1640
1641 1641 def validate_stb(stb):
1642 1642 """validate structured traceback return type
1643 1643
1644 1644 return type of CustomTB *should* be a list of strings, but allow
1645 1645 single strings or None, which are harmless.
1646 1646
1647 1647 This function will *always* return a list of strings,
1648 1648 and will raise a TypeError if stb is inappropriate.
1649 1649 """
1650 1650 msg = "CustomTB must return list of strings, not %r" % stb
1651 1651 if stb is None:
1652 1652 return []
1653 1653 elif isinstance(stb, str):
1654 1654 return [stb]
1655 1655 elif not isinstance(stb, list):
1656 1656 raise TypeError(msg)
1657 1657 # it's a list
1658 1658 for line in stb:
1659 1659 # check every element
1660 1660 if not isinstance(line, str):
1661 1661 raise TypeError(msg)
1662 1662 return stb
1663 1663
1664 1664 if handler is None:
1665 1665 wrapped = dummy_handler
1666 1666 else:
1667 1667 def wrapped(self,etype,value,tb,tb_offset=None):
1668 1668 """wrap CustomTB handler, to protect IPython from user code
1669 1669
1670 1670 This makes it harder (but not impossible) for custom exception
1671 1671 handlers to crash IPython.
1672 1672 """
1673 1673 try:
1674 1674 stb = handler(self,etype,value,tb,tb_offset=tb_offset)
1675 1675 return validate_stb(stb)
1676 1676 except:
1677 1677 # clear custom handler immediately
1678 1678 self.set_custom_exc((), None)
1679 1679 print("Custom TB Handler failed, unregistering", file=sys.stderr)
1680 1680 # show the exception in handler first
1681 1681 stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
1682 1682 print(self.InteractiveTB.stb2text(stb))
1683 1683 print("The original exception:")
1684 1684 stb = self.InteractiveTB.structured_traceback(
1685 1685 (etype,value,tb), tb_offset=tb_offset
1686 1686 )
1687 1687 return stb
1688 1688
1689 1689 self.CustomTB = types.MethodType(wrapped,self)
1690 1690 self.custom_exceptions = exc_tuple
1691 1691
1692 1692 def excepthook(self, etype, value, tb):
1693 1693 """One more defense for GUI apps that call sys.excepthook.
1694 1694
1695 1695 GUI frameworks like wxPython trap exceptions and call
1696 1696 sys.excepthook themselves. I guess this is a feature that
1697 1697 enables them to keep running after exceptions that would
1698 1698 otherwise kill their mainloop. This is a bother for IPython
1699 1699 which excepts to catch all of the program exceptions with a try:
1700 1700 except: statement.
1701 1701
1702 1702 Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
1703 1703 any app directly invokes sys.excepthook, it will look to the user like
1704 1704 IPython crashed. In order to work around this, we can disable the
1705 1705 CrashHandler and replace it with this excepthook instead, which prints a
1706 1706 regular traceback using our InteractiveTB. In this fashion, apps which
1707 1707 call sys.excepthook will generate a regular-looking exception from
1708 1708 IPython, and the CrashHandler will only be triggered by real IPython
1709 1709 crashes.
1710 1710
1711 1711 This hook should be used sparingly, only in places which are not likely
1712 1712 to be true IPython errors.
1713 1713 """
1714 1714 self.showtraceback((etype, value, tb), tb_offset=0)
1715 1715
1716 1716 def _get_exc_info(self, exc_tuple=None):
1717 1717 """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
1718 1718
1719 1719 Ensures sys.last_type,value,traceback hold the exc_info we found,
1720 1720 from whichever source.
1721 1721
1722 1722 raises ValueError if none of these contain any information
1723 1723 """
1724 1724 if exc_tuple is None:
1725 1725 etype, value, tb = sys.exc_info()
1726 1726 else:
1727 1727 etype, value, tb = exc_tuple
1728 1728
1729 1729 if etype is None:
1730 1730 if hasattr(sys, 'last_type'):
1731 1731 etype, value, tb = sys.last_type, sys.last_value, \
1732 1732 sys.last_traceback
1733 1733
1734 1734 if etype is None:
1735 1735 raise ValueError("No exception to find")
1736 1736
1737 1737 # Now store the exception info in sys.last_type etc.
1738 1738 # WARNING: these variables are somewhat deprecated and not
1739 1739 # necessarily safe to use in a threaded environment, but tools
1740 1740 # like pdb depend on their existence, so let's set them. If we
1741 1741 # find problems in the field, we'll need to revisit their use.
1742 1742 sys.last_type = etype
1743 1743 sys.last_value = value
1744 1744 sys.last_traceback = tb
1745 1745
1746 1746 return etype, value, tb
1747 1747
1748 1748 def show_usage_error(self, exc):
1749 1749 """Show a short message for UsageErrors
1750 1750
1751 1751 These are special exceptions that shouldn't show a traceback.
1752 1752 """
1753 1753 print("UsageError: %s" % exc, file=sys.stderr)
1754 1754
1755 1755 def get_exception_only(self, exc_tuple=None):
1756 1756 """
1757 1757 Return as a string (ending with a newline) the exception that
1758 1758 just occurred, without any traceback.
1759 1759 """
1760 1760 etype, value, tb = self._get_exc_info(exc_tuple)
1761 1761 msg = traceback.format_exception_only(etype, value)
1762 1762 return ''.join(msg)
1763 1763
1764 1764 def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None,
1765 1765 exception_only=False):
1766 1766 """Display the exception that just occurred.
1767 1767
1768 1768 If nothing is known about the exception, this is the method which
1769 1769 should be used throughout the code for presenting user tracebacks,
1770 1770 rather than directly invoking the InteractiveTB object.
1771 1771
1772 1772 A specific showsyntaxerror() also exists, but this method can take
1773 1773 care of calling it if needed, so unless you are explicitly catching a
1774 1774 SyntaxError exception, don't try to analyze the stack manually and
1775 1775 simply call this method."""
1776 1776
1777 1777 try:
1778 1778 try:
1779 1779 etype, value, tb = self._get_exc_info(exc_tuple)
1780 1780 except ValueError:
1781 1781 print('No traceback available to show.', file=sys.stderr)
1782 1782 return
1783 1783
1784 1784 if issubclass(etype, SyntaxError):
1785 1785 # Though this won't be called by syntax errors in the input
1786 1786 # line, there may be SyntaxError cases with imported code.
1787 1787 self.showsyntaxerror(filename)
1788 1788 elif etype is UsageError:
1789 1789 self.show_usage_error(value)
1790 1790 else:
1791 1791 if exception_only:
1792 1792 stb = ['An exception has occurred, use %tb to see '
1793 1793 'the full traceback.\n']
1794 1794 stb.extend(self.InteractiveTB.get_exception_only(etype,
1795 1795 value))
1796 1796 else:
1797 1797 try:
1798 1798 # Exception classes can customise their traceback - we
1799 1799 # use this in IPython.parallel for exceptions occurring
1800 1800 # in the engines. This should return a list of strings.
1801 1801 stb = value._render_traceback_()
1802 1802 except Exception:
1803 1803 stb = self.InteractiveTB.structured_traceback(etype,
1804 1804 value, tb, tb_offset=tb_offset)
1805 1805
1806 1806 self._showtraceback(etype, value, stb)
1807 1807 if self.call_pdb:
1808 1808 # drop into debugger
1809 1809 self.debugger(force=True)
1810 1810 return
1811 1811
1812 1812 # Actually show the traceback
1813 1813 self._showtraceback(etype, value, stb)
1814 1814
1815 1815 except KeyboardInterrupt:
1816 1816 print('\n' + self.get_exception_only(), file=sys.stderr)
1817 1817
1818 1818 def _showtraceback(self, etype, evalue, stb):
1819 1819 """Actually show a traceback.
1820 1820
1821 1821 Subclasses may override this method to put the traceback on a different
1822 1822 place, like a side channel.
1823 1823 """
1824 1824 print(self.InteractiveTB.stb2text(stb))
1825 1825
1826 1826 def showsyntaxerror(self, filename=None):
1827 1827 """Display the syntax error that just occurred.
1828 1828
1829 1829 This doesn't display a stack trace because there isn't one.
1830 1830
1831 1831 If a filename is given, it is stuffed in the exception instead
1832 1832 of what was there before (because Python's parser always uses
1833 1833 "<string>" when reading from a string).
1834 1834 """
1835 1835 etype, value, last_traceback = self._get_exc_info()
1836 1836
1837 1837 if filename and issubclass(etype, SyntaxError):
1838 1838 try:
1839 1839 value.filename = filename
1840 1840 except:
1841 1841 # Not the format we expect; leave it alone
1842 1842 pass
1843 1843
1844 1844 stb = self.SyntaxTB.structured_traceback(etype, value, [])
1845 1845 self._showtraceback(etype, value, stb)
1846 1846
1847 1847 # This is overridden in TerminalInteractiveShell to show a message about
1848 1848 # the %paste magic.
1849 1849 def showindentationerror(self):
1850 1850 """Called by run_cell when there's an IndentationError in code entered
1851 1851 at the prompt.
1852 1852
1853 1853 This is overridden in TerminalInteractiveShell to show a message about
1854 1854 the %paste magic."""
1855 1855 self.showsyntaxerror()
1856 1856
1857 1857 #-------------------------------------------------------------------------
1858 1858 # Things related to readline
1859 1859 #-------------------------------------------------------------------------
1860 1860
1861 1861 def init_readline(self):
1862 1862 """DEPRECATED
1863 1863
1864 1864 Moved to terminal subclass, here only to simplify the init logic."""
1865 1865 # Set a number of methods that depend on readline to be no-op
1866 1866 warnings.warn('`init_readline` is no-op since IPython 5.0 and is Deprecated',
1867 1867 DeprecationWarning, stacklevel=2)
1868 1868 self.set_custom_completer = no_op
1869 1869
1870 1870 @skip_doctest
1871 1871 def set_next_input(self, s, replace=False):
1872 1872 """ Sets the 'default' input string for the next command line.
1873 1873
1874 1874 Example::
1875 1875
1876 1876 In [1]: _ip.set_next_input("Hello Word")
1877 1877 In [2]: Hello Word_ # cursor is here
1878 1878 """
1879 1879 self.rl_next_input = py3compat.cast_bytes_py2(s)
1880 1880
1881 1881 def _indent_current_str(self):
1882 1882 """return the current level of indentation as a string"""
1883 1883 return self.input_splitter.indent_spaces * ' '
1884 1884
1885 1885 #-------------------------------------------------------------------------
1886 1886 # Things related to text completion
1887 1887 #-------------------------------------------------------------------------
1888 1888
1889 1889 def init_completer(self):
1890 1890 """Initialize the completion machinery.
1891 1891
1892 1892 This creates completion machinery that can be used by client code,
1893 1893 either interactively in-process (typically triggered by the readline
1894 1894 library), programmatically (such as in test suites) or out-of-process
1895 1895 (typically over the network by remote frontends).
1896 1896 """
1897 1897 from IPython.core.completer import IPCompleter
1898 1898 from IPython.core.completerlib import (module_completer,
1899 1899 magic_run_completer, cd_completer, reset_completer)
1900 1900
1901 1901 self.Completer = IPCompleter(shell=self,
1902 1902 namespace=self.user_ns,
1903 1903 global_namespace=self.user_global_ns,
1904 1904 parent=self,
1905 1905 )
1906 1906 self.configurables.append(self.Completer)
1907 1907
1908 1908 # Add custom completers to the basic ones built into IPCompleter
1909 1909 sdisp = self.strdispatchers.get('complete_command', StrDispatch())
1910 1910 self.strdispatchers['complete_command'] = sdisp
1911 1911 self.Completer.custom_completers = sdisp
1912 1912
1913 1913 self.set_hook('complete_command', module_completer, str_key = 'import')
1914 1914 self.set_hook('complete_command', module_completer, str_key = 'from')
1915 1915 self.set_hook('complete_command', module_completer, str_key = '%aimport')
1916 1916 self.set_hook('complete_command', magic_run_completer, str_key = '%run')
1917 1917 self.set_hook('complete_command', cd_completer, str_key = '%cd')
1918 1918 self.set_hook('complete_command', reset_completer, str_key = '%reset')
1919 1919
1920 1920
1921 1921 def complete(self, text, line=None, cursor_pos=None):
1922 1922 """Return the completed text and a list of completions.
1923 1923
1924 1924 Parameters
1925 1925 ----------
1926 1926
1927 1927 text : string
1928 1928 A string of text to be completed on. It can be given as empty and
1929 1929 instead a line/position pair are given. In this case, the
1930 1930 completer itself will split the line like readline does.
1931 1931
1932 1932 line : string, optional
1933 1933 The complete line that text is part of.
1934 1934
1935 1935 cursor_pos : int, optional
1936 1936 The position of the cursor on the input line.
1937 1937
1938 1938 Returns
1939 1939 -------
1940 1940 text : string
1941 1941 The actual text that was completed.
1942 1942
1943 1943 matches : list
1944 1944 A sorted list with all possible completions.
1945 1945
1946 1946 The optional arguments allow the completion to take more context into
1947 1947 account, and are part of the low-level completion API.
1948 1948
1949 1949 This is a wrapper around the completion mechanism, similar to what
1950 1950 readline does at the command line when the TAB key is hit. By
1951 1951 exposing it as a method, it can be used by other non-readline
1952 1952 environments (such as GUIs) for text completion.
1953 1953
1954 1954 Simple usage example:
1955 1955
1956 1956 In [1]: x = 'hello'
1957 1957
1958 1958 In [2]: _ip.complete('x.l')
1959 1959 Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
1960 1960 """
1961 1961
1962 1962 # Inject names into __builtin__ so we can complete on the added names.
1963 1963 with self.builtin_trap:
1964 1964 return self.Completer.complete(text, line, cursor_pos)
1965 1965
1966 1966 def set_custom_completer(self, completer, pos=0):
1967 1967 """Adds a new custom completer function.
1968 1968
1969 1969 The position argument (defaults to 0) is the index in the completers
1970 1970 list where you want the completer to be inserted."""
1971 1971
1972 1972 newcomp = types.MethodType(completer,self.Completer)
1973 1973 self.Completer.matchers.insert(pos,newcomp)
1974 1974
1975 1975 def set_completer_frame(self, frame=None):
1976 1976 """Set the frame of the completer."""
1977 1977 if frame:
1978 1978 self.Completer.namespace = frame.f_locals
1979 1979 self.Completer.global_namespace = frame.f_globals
1980 1980 else:
1981 1981 self.Completer.namespace = self.user_ns
1982 1982 self.Completer.global_namespace = self.user_global_ns
1983 1983
1984 1984 #-------------------------------------------------------------------------
1985 1985 # Things related to magics
1986 1986 #-------------------------------------------------------------------------
1987 1987
1988 1988 def init_magics(self):
1989 1989 from IPython.core import magics as m
1990 1990 self.magics_manager = magic.MagicsManager(shell=self,
1991 1991 parent=self,
1992 1992 user_magics=m.UserMagics(self))
1993 1993 self.configurables.append(self.magics_manager)
1994 1994
1995 1995 # Expose as public API from the magics manager
1996 1996 self.register_magics = self.magics_manager.register
1997 1997
1998 1998 self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
1999 1999 m.ConfigMagics, m.DisplayMagics, m.ExecutionMagics,
2000 2000 m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
2001 2001 m.NamespaceMagics, m.OSMagics, m.PylabMagics, m.ScriptMagics,
2002 2002 )
2003 2003
2004 2004 # Register Magic Aliases
2005 2005 mman = self.magics_manager
2006 2006 # FIXME: magic aliases should be defined by the Magics classes
2007 2007 # or in MagicsManager, not here
2008 2008 mman.register_alias('ed', 'edit')
2009 2009 mman.register_alias('hist', 'history')
2010 2010 mman.register_alias('rep', 'recall')
2011 2011 mman.register_alias('SVG', 'svg', 'cell')
2012 2012 mman.register_alias('HTML', 'html', 'cell')
2013 2013 mman.register_alias('file', 'writefile', 'cell')
2014 2014
2015 2015 # FIXME: Move the color initialization to the DisplayHook, which
2016 2016 # should be split into a prompt manager and displayhook. We probably
2017 2017 # even need a centralize colors management object.
2018 2018 self.magic('colors %s' % self.colors)
2019 2019
2020 2020 # Defined here so that it's included in the documentation
2021 2021 @functools.wraps(magic.MagicsManager.register_function)
2022 2022 def register_magic_function(self, func, magic_kind='line', magic_name=None):
2023 2023 self.magics_manager.register_function(func,
2024 2024 magic_kind=magic_kind, magic_name=magic_name)
2025 2025
2026 2026 def run_line_magic(self, magic_name, line):
2027 2027 """Execute the given line magic.
2028 2028
2029 2029 Parameters
2030 2030 ----------
2031 2031 magic_name : str
2032 2032 Name of the desired magic function, without '%' prefix.
2033 2033
2034 2034 line : str
2035 2035 The rest of the input line as a single string.
2036 2036 """
2037 2037 fn = self.find_line_magic(magic_name)
2038 2038 if fn is None:
2039 2039 cm = self.find_cell_magic(magic_name)
2040 2040 etpl = "Line magic function `%%%s` not found%s."
2041 2041 extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
2042 2042 'did you mean that instead?)' % magic_name )
2043 2043 error(etpl % (magic_name, extra))
2044 2044 else:
2045 2045 # Note: this is the distance in the stack to the user's frame.
2046 2046 # This will need to be updated if the internal calling logic gets
2047 2047 # refactored, or else we'll be expanding the wrong variables.
2048 2048 stack_depth = 2
2049 2049 magic_arg_s = self.var_expand(line, stack_depth)
2050 2050 # Put magic args in a list so we can call with f(*a) syntax
2051 2051 args = [magic_arg_s]
2052 2052 kwargs = {}
2053 2053 # Grab local namespace if we need it:
2054 2054 if getattr(fn, "needs_local_scope", False):
2055 2055 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
2056 2056 with self.builtin_trap:
2057 2057 result = fn(*args,**kwargs)
2058 2058 return result
2059 2059
2060 2060 def run_cell_magic(self, magic_name, line, cell):
2061 2061 """Execute the given cell magic.
2062 2062
2063 2063 Parameters
2064 2064 ----------
2065 2065 magic_name : str
2066 2066 Name of the desired magic function, without '%' prefix.
2067 2067
2068 2068 line : str
2069 2069 The rest of the first input line as a single string.
2070 2070
2071 2071 cell : str
2072 2072 The body of the cell as a (possibly multiline) string.
2073 2073 """
2074 2074 fn = self.find_cell_magic(magic_name)
2075 2075 if fn is None:
2076 2076 lm = self.find_line_magic(magic_name)
2077 2077 etpl = "Cell magic `%%{0}` not found{1}."
2078 2078 extra = '' if lm is None else (' (But line magic `%{0}` exists, '
2079 2079 'did you mean that instead?)'.format(magic_name))
2080 2080 error(etpl.format(magic_name, extra))
2081 2081 elif cell == '':
2082 2082 message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
2083 2083 if self.find_line_magic(magic_name) is not None:
2084 2084 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
2085 2085 raise UsageError(message)
2086 2086 else:
2087 2087 # Note: this is the distance in the stack to the user's frame.
2088 2088 # This will need to be updated if the internal calling logic gets
2089 2089 # refactored, or else we'll be expanding the wrong variables.
2090 2090 stack_depth = 2
2091 2091 magic_arg_s = self.var_expand(line, stack_depth)
2092 2092 with self.builtin_trap:
2093 2093 result = fn(magic_arg_s, cell)
2094 2094 return result
2095 2095
2096 2096 def find_line_magic(self, magic_name):
2097 2097 """Find and return a line magic by name.
2098 2098
2099 2099 Returns None if the magic isn't found."""
2100 2100 return self.magics_manager.magics['line'].get(magic_name)
2101 2101
2102 2102 def find_cell_magic(self, magic_name):
2103 2103 """Find and return a cell magic by name.
2104 2104
2105 2105 Returns None if the magic isn't found."""
2106 2106 return self.magics_manager.magics['cell'].get(magic_name)
2107 2107
2108 2108 def find_magic(self, magic_name, magic_kind='line'):
2109 2109 """Find and return a magic of the given type by name.
2110 2110
2111 2111 Returns None if the magic isn't found."""
2112 2112 return self.magics_manager.magics[magic_kind].get(magic_name)
2113 2113
2114 2114 def magic(self, arg_s):
2115 2115 """DEPRECATED. Use run_line_magic() instead.
2116 2116
2117 2117 Call a magic function by name.
2118 2118
2119 2119 Input: a string containing the name of the magic function to call and
2120 2120 any additional arguments to be passed to the magic.
2121 2121
2122 2122 magic('name -opt foo bar') is equivalent to typing at the ipython
2123 2123 prompt:
2124 2124
2125 2125 In[1]: %name -opt foo bar
2126 2126
2127 2127 To call a magic without arguments, simply use magic('name').
2128 2128
2129 2129 This provides a proper Python function to call IPython's magics in any
2130 2130 valid Python code you can type at the interpreter, including loops and
2131 2131 compound statements.
2132 2132 """
2133 2133 # TODO: should we issue a loud deprecation warning here?
2134 2134 magic_name, _, magic_arg_s = arg_s.partition(' ')
2135 2135 magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
2136 2136 return self.run_line_magic(magic_name, magic_arg_s)
2137 2137
2138 2138 #-------------------------------------------------------------------------
2139 2139 # Things related to macros
2140 2140 #-------------------------------------------------------------------------
2141 2141
2142 2142 def define_macro(self, name, themacro):
2143 2143 """Define a new macro
2144 2144
2145 2145 Parameters
2146 2146 ----------
2147 2147 name : str
2148 2148 The name of the macro.
2149 2149 themacro : str or Macro
2150 2150 The action to do upon invoking the macro. If a string, a new
2151 2151 Macro object is created by passing the string to it.
2152 2152 """
2153 2153
2154 2154 from IPython.core import macro
2155 2155
2156 2156 if isinstance(themacro, str):
2157 2157 themacro = macro.Macro(themacro)
2158 2158 if not isinstance(themacro, macro.Macro):
2159 2159 raise ValueError('A macro must be a string or a Macro instance.')
2160 2160 self.user_ns[name] = themacro
2161 2161
2162 2162 #-------------------------------------------------------------------------
2163 2163 # Things related to the running of system commands
2164 2164 #-------------------------------------------------------------------------
2165 2165
2166 2166 def system_piped(self, cmd):
2167 2167 """Call the given cmd in a subprocess, piping stdout/err
2168 2168
2169 2169 Parameters
2170 2170 ----------
2171 2171 cmd : str
2172 2172 Command to execute (can not end in '&', as background processes are
2173 2173 not supported. Should not be a command that expects input
2174 2174 other than simple text.
2175 2175 """
2176 2176 if cmd.rstrip().endswith('&'):
2177 2177 # this is *far* from a rigorous test
2178 2178 # We do not support backgrounding processes because we either use
2179 2179 # pexpect or pipes to read from. Users can always just call
2180 2180 # os.system() or use ip.system=ip.system_raw
2181 2181 # if they really want a background process.
2182 2182 raise OSError("Background processes not supported.")
2183 2183
2184 2184 # we explicitly do NOT return the subprocess status code, because
2185 2185 # a non-None value would trigger :func:`sys.displayhook` calls.
2186 2186 # Instead, we store the exit_code in user_ns.
2187 2187 self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
2188 2188
2189 2189 def system_raw(self, cmd):
2190 2190 """Call the given cmd in a subprocess using os.system on Windows or
2191 2191 subprocess.call using the system shell on other platforms.
2192 2192
2193 2193 Parameters
2194 2194 ----------
2195 2195 cmd : str
2196 2196 Command to execute.
2197 2197 """
2198 2198 cmd = self.var_expand(cmd, depth=1)
2199 2199 # protect os.system from UNC paths on Windows, which it can't handle:
2200 2200 if sys.platform == 'win32':
2201 2201 from IPython.utils._process_win32 import AvoidUNCPath
2202 2202 with AvoidUNCPath() as path:
2203 2203 if path is not None:
2204 2204 cmd = '"pushd %s &&"%s' % (path, cmd)
2205 2205 try:
2206 2206 ec = os.system(cmd)
2207 2207 except KeyboardInterrupt:
2208 2208 print('\n' + self.get_exception_only(), file=sys.stderr)
2209 2209 ec = -2
2210 2210 else:
2211 2211 # For posix the result of the subprocess.call() below is an exit
2212 2212 # code, which by convention is zero for success, positive for
2213 2213 # program failure. Exit codes above 128 are reserved for signals,
2214 2214 # and the formula for converting a signal to an exit code is usually
2215 2215 # signal_number+128. To more easily differentiate between exit
2216 2216 # codes and signals, ipython uses negative numbers. For instance
2217 2217 # since control-c is signal 2 but exit code 130, ipython's
2218 2218 # _exit_code variable will read -2. Note that some shells like
2219 2219 # csh and fish don't follow sh/bash conventions for exit codes.
2220 2220 executable = os.environ.get('SHELL', None)
2221 2221 try:
2222 2222 # Use env shell instead of default /bin/sh
2223 2223 ec = subprocess.call(cmd, shell=True, executable=executable)
2224 2224 except KeyboardInterrupt:
2225 2225 # intercept control-C; a long traceback is not useful here
2226 2226 print('\n' + self.get_exception_only(), file=sys.stderr)
2227 2227 ec = 130
2228 2228 if ec > 128:
2229 2229 ec = -(ec - 128)
2230 2230
2231 2231 # We explicitly do NOT return the subprocess status code, because
2232 2232 # a non-None value would trigger :func:`sys.displayhook` calls.
2233 2233 # Instead, we store the exit_code in user_ns. Note the semantics
2234 2234 # of _exit_code: for control-c, _exit_code == -signal.SIGNIT,
2235 2235 # but raising SystemExit(_exit_code) will give status 254!
2236 2236 self.user_ns['_exit_code'] = ec
2237 2237
2238 2238 # use piped system by default, because it is better behaved
2239 2239 system = system_piped
2240 2240
2241 2241 def getoutput(self, cmd, split=True, depth=0):
2242 2242 """Get output (possibly including stderr) from a subprocess.
2243 2243
2244 2244 Parameters
2245 2245 ----------
2246 2246 cmd : str
2247 2247 Command to execute (can not end in '&', as background processes are
2248 2248 not supported.
2249 2249 split : bool, optional
2250 2250 If True, split the output into an IPython SList. Otherwise, an
2251 2251 IPython LSString is returned. These are objects similar to normal
2252 2252 lists and strings, with a few convenience attributes for easier
2253 2253 manipulation of line-based output. You can use '?' on them for
2254 2254 details.
2255 2255 depth : int, optional
2256 2256 How many frames above the caller are the local variables which should
2257 2257 be expanded in the command string? The default (0) assumes that the
2258 2258 expansion variables are in the stack frame calling this function.
2259 2259 """
2260 2260 if cmd.rstrip().endswith('&'):
2261 2261 # this is *far* from a rigorous test
2262 2262 raise OSError("Background processes not supported.")
2263 2263 out = getoutput(self.var_expand(cmd, depth=depth+1))
2264 2264 if split:
2265 2265 out = SList(out.splitlines())
2266 2266 else:
2267 2267 out = LSString(out)
2268 2268 return out
2269 2269
2270 2270 #-------------------------------------------------------------------------
2271 2271 # Things related to aliases
2272 2272 #-------------------------------------------------------------------------
2273 2273
2274 2274 def init_alias(self):
2275 2275 self.alias_manager = AliasManager(shell=self, parent=self)
2276 2276 self.configurables.append(self.alias_manager)
2277 2277
2278 2278 #-------------------------------------------------------------------------
2279 2279 # Things related to extensions
2280 2280 #-------------------------------------------------------------------------
2281 2281
2282 2282 def init_extension_manager(self):
2283 2283 self.extension_manager = ExtensionManager(shell=self, parent=self)
2284 2284 self.configurables.append(self.extension_manager)
2285 2285
2286 2286 #-------------------------------------------------------------------------
2287 2287 # Things related to payloads
2288 2288 #-------------------------------------------------------------------------
2289 2289
2290 2290 def init_payload(self):
2291 2291 self.payload_manager = PayloadManager(parent=self)
2292 2292 self.configurables.append(self.payload_manager)
2293 2293
2294 2294 #-------------------------------------------------------------------------
2295 2295 # Things related to the prefilter
2296 2296 #-------------------------------------------------------------------------
2297 2297
2298 2298 def init_prefilter(self):
2299 2299 self.prefilter_manager = PrefilterManager(shell=self, parent=self)
2300 2300 self.configurables.append(self.prefilter_manager)
2301 2301 # Ultimately this will be refactored in the new interpreter code, but
2302 2302 # for now, we should expose the main prefilter method (there's legacy
2303 2303 # code out there that may rely on this).
2304 2304 self.prefilter = self.prefilter_manager.prefilter_lines
2305 2305
2306 2306 def auto_rewrite_input(self, cmd):
2307 2307 """Print to the screen the rewritten form of the user's command.
2308 2308
2309 2309 This shows visual feedback by rewriting input lines that cause
2310 2310 automatic calling to kick in, like::
2311 2311
2312 2312 /f x
2313 2313
2314 2314 into::
2315 2315
2316 2316 ------> f(x)
2317 2317
2318 2318 after the user's input prompt. This helps the user understand that the
2319 2319 input line was transformed automatically by IPython.
2320 2320 """
2321 2321 if not self.show_rewritten_input:
2322 2322 return
2323 2323
2324 2324 # This is overridden in TerminalInteractiveShell to use fancy prompts
2325 2325 print("------> " + cmd)
2326 2326
2327 2327 #-------------------------------------------------------------------------
2328 2328 # Things related to extracting values/expressions from kernel and user_ns
2329 2329 #-------------------------------------------------------------------------
2330 2330
2331 2331 def _user_obj_error(self):
2332 2332 """return simple exception dict
2333 2333
2334 2334 for use in user_expressions
2335 2335 """
2336 2336
2337 2337 etype, evalue, tb = self._get_exc_info()
2338 2338 stb = self.InteractiveTB.get_exception_only(etype, evalue)
2339 2339
2340 2340 exc_info = {
2341 2341 u'status' : 'error',
2342 2342 u'traceback' : stb,
2343 2343 u'ename' : etype.__name__,
2344 2344 u'evalue' : py3compat.safe_unicode(evalue),
2345 2345 }
2346 2346
2347 2347 return exc_info
2348 2348
2349 2349 def _format_user_obj(self, obj):
2350 2350 """format a user object to display dict
2351 2351
2352 2352 for use in user_expressions
2353 2353 """
2354 2354
2355 2355 data, md = self.display_formatter.format(obj)
2356 2356 value = {
2357 2357 'status' : 'ok',
2358 2358 'data' : data,
2359 2359 'metadata' : md,
2360 2360 }
2361 2361 return value
2362 2362
2363 2363 def user_expressions(self, expressions):
2364 2364 """Evaluate a dict of expressions in the user's namespace.
2365 2365
2366 2366 Parameters
2367 2367 ----------
2368 2368 expressions : dict
2369 2369 A dict with string keys and string values. The expression values
2370 2370 should be valid Python expressions, each of which will be evaluated
2371 2371 in the user namespace.
2372 2372
2373 2373 Returns
2374 2374 -------
2375 2375 A dict, keyed like the input expressions dict, with the rich mime-typed
2376 2376 display_data of each value.
2377 2377 """
2378 2378 out = {}
2379 2379 user_ns = self.user_ns
2380 2380 global_ns = self.user_global_ns
2381 2381
2382 2382 for key, expr in expressions.items():
2383 2383 try:
2384 2384 value = self._format_user_obj(eval(expr, global_ns, user_ns))
2385 2385 except:
2386 2386 value = self._user_obj_error()
2387 2387 out[key] = value
2388 2388 return out
2389 2389
2390 2390 #-------------------------------------------------------------------------
2391 2391 # Things related to the running of code
2392 2392 #-------------------------------------------------------------------------
2393 2393
2394 2394 def ex(self, cmd):
2395 2395 """Execute a normal python statement in user namespace."""
2396 2396 with self.builtin_trap:
2397 2397 exec(cmd, self.user_global_ns, self.user_ns)
2398 2398
2399 2399 def ev(self, expr):
2400 2400 """Evaluate python expression expr in user namespace.
2401 2401
2402 2402 Returns the result of evaluation
2403 2403 """
2404 2404 with self.builtin_trap:
2405 2405 return eval(expr, self.user_global_ns, self.user_ns)
2406 2406
2407 2407 def safe_execfile(self, fname, *where, **kw):
2408 2408 """A safe version of the builtin execfile().
2409 2409
2410 2410 This version will never throw an exception, but instead print
2411 2411 helpful error messages to the screen. This only works on pure
2412 2412 Python files with the .py extension.
2413 2413
2414 2414 Parameters
2415 2415 ----------
2416 2416 fname : string
2417 2417 The name of the file to be executed.
2418 2418 where : tuple
2419 2419 One or two namespaces, passed to execfile() as (globals,locals).
2420 2420 If only one is given, it is passed as both.
2421 2421 exit_ignore : bool (False)
2422 2422 If True, then silence SystemExit for non-zero status (it is always
2423 2423 silenced for zero status, as it is so common).
2424 2424 raise_exceptions : bool (False)
2425 2425 If True raise exceptions everywhere. Meant for testing.
2426 2426 shell_futures : bool (False)
2427 2427 If True, the code will share future statements with the interactive
2428 2428 shell. It will both be affected by previous __future__ imports, and
2429 2429 any __future__ imports in the code will affect the shell. If False,
2430 2430 __future__ imports are not shared in either direction.
2431 2431
2432 2432 """
2433 2433 kw.setdefault('exit_ignore', False)
2434 2434 kw.setdefault('raise_exceptions', False)
2435 2435 kw.setdefault('shell_futures', False)
2436 2436
2437 2437 fname = os.path.abspath(os.path.expanduser(fname))
2438 2438
2439 2439 # Make sure we can open the file
2440 2440 try:
2441 2441 with open(fname):
2442 2442 pass
2443 2443 except:
2444 2444 warn('Could not open file <%s> for safe execution.' % fname)
2445 2445 return
2446 2446
2447 2447 # Find things also in current directory. This is needed to mimic the
2448 2448 # behavior of running a script from the system command line, where
2449 2449 # Python inserts the script's directory into sys.path
2450 2450 dname = os.path.dirname(fname)
2451 2451
2452 2452 with prepended_to_syspath(dname), self.builtin_trap:
2453 2453 try:
2454 2454 glob, loc = (where + (None, ))[:2]
2455 2455 py3compat.execfile(
2456 2456 fname, glob, loc,
2457 2457 self.compile if kw['shell_futures'] else None)
2458 2458 except SystemExit as status:
2459 2459 # If the call was made with 0 or None exit status (sys.exit(0)
2460 2460 # or sys.exit() ), don't bother showing a traceback, as both of
2461 2461 # these are considered normal by the OS:
2462 2462 # > python -c'import sys;sys.exit(0)'; echo $?
2463 2463 # 0
2464 2464 # > python -c'import sys;sys.exit()'; echo $?
2465 2465 # 0
2466 2466 # For other exit status, we show the exception unless
2467 2467 # explicitly silenced, but only in short form.
2468 2468 if status.code:
2469 2469 if kw['raise_exceptions']:
2470 2470 raise
2471 2471 if not kw['exit_ignore']:
2472 2472 self.showtraceback(exception_only=True)
2473 2473 except:
2474 2474 if kw['raise_exceptions']:
2475 2475 raise
2476 2476 # tb offset is 2 because we wrap execfile
2477 2477 self.showtraceback(tb_offset=2)
2478 2478
2479 2479 def safe_execfile_ipy(self, fname, shell_futures=False, raise_exceptions=False):
2480 2480 """Like safe_execfile, but for .ipy or .ipynb files with IPython syntax.
2481 2481
2482 2482 Parameters
2483 2483 ----------
2484 2484 fname : str
2485 2485 The name of the file to execute. The filename must have a
2486 2486 .ipy or .ipynb extension.
2487 2487 shell_futures : bool (False)
2488 2488 If True, the code will share future statements with the interactive
2489 2489 shell. It will both be affected by previous __future__ imports, and
2490 2490 any __future__ imports in the code will affect the shell. If False,
2491 2491 __future__ imports are not shared in either direction.
2492 2492 raise_exceptions : bool (False)
2493 2493 If True raise exceptions everywhere. Meant for testing.
2494 2494 """
2495 2495 fname = os.path.abspath(os.path.expanduser(fname))
2496 2496
2497 2497 # Make sure we can open the file
2498 2498 try:
2499 2499 with open(fname):
2500 2500 pass
2501 2501 except:
2502 2502 warn('Could not open file <%s> for safe execution.' % fname)
2503 2503 return
2504 2504
2505 2505 # Find things also in current directory. This is needed to mimic the
2506 2506 # behavior of running a script from the system command line, where
2507 2507 # Python inserts the script's directory into sys.path
2508 2508 dname = os.path.dirname(fname)
2509 2509
2510 2510 def get_cells():
2511 2511 """generator for sequence of code blocks to run"""
2512 2512 if fname.endswith('.ipynb'):
2513 2513 from nbformat import read
2514 2514 with io_open(fname) as f:
2515 2515 nb = read(f, as_version=4)
2516 2516 if not nb.cells:
2517 2517 return
2518 2518 for cell in nb.cells:
2519 2519 if cell.cell_type == 'code':
2520 2520 yield cell.source
2521 2521 else:
2522 2522 with open(fname) as f:
2523 2523 yield f.read()
2524 2524
2525 2525 with prepended_to_syspath(dname):
2526 2526 try:
2527 2527 for cell in get_cells():
2528 2528 result = self.run_cell(cell, silent=True, shell_futures=shell_futures)
2529 2529 if raise_exceptions:
2530 2530 result.raise_error()
2531 2531 elif not result.success:
2532 2532 break
2533 2533 except:
2534 2534 if raise_exceptions:
2535 2535 raise
2536 2536 self.showtraceback()
2537 2537 warn('Unknown failure executing file: <%s>' % fname)
2538 2538
2539 2539 def safe_run_module(self, mod_name, where):
2540 2540 """A safe version of runpy.run_module().
2541 2541
2542 2542 This version will never throw an exception, but instead print
2543 2543 helpful error messages to the screen.
2544 2544
2545 2545 `SystemExit` exceptions with status code 0 or None are ignored.
2546 2546
2547 2547 Parameters
2548 2548 ----------
2549 2549 mod_name : string
2550 2550 The name of the module to be executed.
2551 2551 where : dict
2552 2552 The globals namespace.
2553 2553 """
2554 2554 try:
2555 2555 try:
2556 2556 where.update(
2557 2557 runpy.run_module(str(mod_name), run_name="__main__",
2558 2558 alter_sys=True)
2559 2559 )
2560 2560 except SystemExit as status:
2561 2561 if status.code:
2562 2562 raise
2563 2563 except:
2564 2564 self.showtraceback()
2565 2565 warn('Unknown failure executing module: <%s>' % mod_name)
2566 2566
2567 2567 def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True):
2568 2568 """Run a complete IPython cell.
2569 2569
2570 2570 Parameters
2571 2571 ----------
2572 2572 raw_cell : str
2573 2573 The code (including IPython code such as %magic functions) to run.
2574 2574 store_history : bool
2575 2575 If True, the raw and translated cell will be stored in IPython's
2576 2576 history. For user code calling back into IPython's machinery, this
2577 2577 should be set to False.
2578 2578 silent : bool
2579 2579 If True, avoid side-effects, such as implicit displayhooks and
2580 2580 and logging. silent=True forces store_history=False.
2581 2581 shell_futures : bool
2582 2582 If True, the code will share future statements with the interactive
2583 2583 shell. It will both be affected by previous __future__ imports, and
2584 2584 any __future__ imports in the code will affect the shell. If False,
2585 2585 __future__ imports are not shared in either direction.
2586 2586
2587 2587 Returns
2588 2588 -------
2589 2589 result : :class:`ExecutionResult`
2590 2590 """
2591 2591 result = ExecutionResult()
2592 2592
2593 2593 if (not raw_cell) or raw_cell.isspace():
2594 2594 self.last_execution_succeeded = True
2595 2595 return result
2596 2596
2597 2597 if silent:
2598 2598 store_history = False
2599 2599
2600 2600 if store_history:
2601 2601 result.execution_count = self.execution_count
2602 2602
2603 2603 def error_before_exec(value):
2604 2604 result.error_before_exec = value
2605 2605 self.last_execution_succeeded = False
2606 2606 return result
2607 2607
2608 2608 self.events.trigger('pre_execute')
2609 2609 if not silent:
2610 2610 self.events.trigger('pre_run_cell')
2611 2611
2612 2612 # If any of our input transformation (input_transformer_manager or
2613 2613 # prefilter_manager) raises an exception, we store it in this variable
2614 2614 # so that we can display the error after logging the input and storing
2615 2615 # it in the history.
2616 2616 preprocessing_exc_tuple = None
2617 2617 try:
2618 2618 # Static input transformations
2619 2619 cell = self.input_transformer_manager.transform_cell(raw_cell)
2620 2620 except SyntaxError:
2621 2621 preprocessing_exc_tuple = sys.exc_info()
2622 2622 cell = raw_cell # cell has to exist so it can be stored/logged
2623 2623 else:
2624 2624 if len(cell.splitlines()) == 1:
2625 2625 # Dynamic transformations - only applied for single line commands
2626 2626 with self.builtin_trap:
2627 2627 try:
2628 2628 # use prefilter_lines to handle trailing newlines
2629 2629 # restore trailing newline for ast.parse
2630 2630 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
2631 2631 except Exception:
2632 2632 # don't allow prefilter errors to crash IPython
2633 2633 preprocessing_exc_tuple = sys.exc_info()
2634 2634
2635 2635 # Store raw and processed history
2636 2636 if store_history:
2637 2637 self.history_manager.store_inputs(self.execution_count,
2638 2638 cell, raw_cell)
2639 2639 if not silent:
2640 2640 self.logger.log(cell, raw_cell)
2641 2641
2642 2642 # Display the exception if input processing failed.
2643 2643 if preprocessing_exc_tuple is not None:
2644 2644 self.showtraceback(preprocessing_exc_tuple)
2645 2645 if store_history:
2646 2646 self.execution_count += 1
2647 2647 return error_before_exec(preprocessing_exc_tuple[2])
2648 2648
2649 2649 # Our own compiler remembers the __future__ environment. If we want to
2650 2650 # run code with a separate __future__ environment, use the default
2651 2651 # compiler
2652 2652 compiler = self.compile if shell_futures else CachingCompiler()
2653 2653
2654 2654 with self.builtin_trap:
2655 2655 cell_name = self.compile.cache(cell, self.execution_count)
2656 2656
2657 2657 with self.display_trap:
2658 2658 # Compile to bytecode
2659 2659 try:
2660 2660 code_ast = compiler.ast_parse(cell, filename=cell_name)
2661 2661 except self.custom_exceptions as e:
2662 2662 etype, value, tb = sys.exc_info()
2663 2663 self.CustomTB(etype, value, tb)
2664 2664 return error_before_exec(e)
2665 2665 except IndentationError as e:
2666 2666 self.showindentationerror()
2667 2667 if store_history:
2668 2668 self.execution_count += 1
2669 2669 return error_before_exec(e)
2670 2670 except (OverflowError, SyntaxError, ValueError, TypeError,
2671 2671 MemoryError) as e:
2672 2672 self.showsyntaxerror()
2673 2673 if store_history:
2674 2674 self.execution_count += 1
2675 2675 return error_before_exec(e)
2676 2676
2677 2677 # Apply AST transformations
2678 2678 try:
2679 2679 code_ast = self.transform_ast(code_ast)
2680 2680 except InputRejected as e:
2681 2681 self.showtraceback()
2682 2682 if store_history:
2683 2683 self.execution_count += 1
2684 2684 return error_before_exec(e)
2685 2685
2686 2686 # Give the displayhook a reference to our ExecutionResult so it
2687 2687 # can fill in the output value.
2688 2688 self.displayhook.exec_result = result
2689 2689
2690 2690 # Execute the user code
2691 2691 interactivity = "none" if silent else self.ast_node_interactivity
2692 2692 has_raised = self.run_ast_nodes(code_ast.body, cell_name,
2693 2693 interactivity=interactivity, compiler=compiler, result=result)
2694 2694
2695 2695 self.last_execution_succeeded = not has_raised
2696 2696
2697 2697 # Reset this so later displayed values do not modify the
2698 2698 # ExecutionResult
2699 2699 self.displayhook.exec_result = None
2700 2700
2701 2701 self.events.trigger('post_execute')
2702 2702 if not silent:
2703 2703 self.events.trigger('post_run_cell')
2704 2704
2705 2705 if store_history:
2706 2706 # Write output to the database. Does nothing unless
2707 2707 # history output logging is enabled.
2708 2708 self.history_manager.store_output(self.execution_count)
2709 2709 # Each cell is a *single* input, regardless of how many lines it has
2710 2710 self.execution_count += 1
2711 2711
2712 2712 return result
2713 2713
2714 2714 def transform_ast(self, node):
2715 2715 """Apply the AST transformations from self.ast_transformers
2716 2716
2717 2717 Parameters
2718 2718 ----------
2719 2719 node : ast.Node
2720 2720 The root node to be transformed. Typically called with the ast.Module
2721 2721 produced by parsing user input.
2722 2722
2723 2723 Returns
2724 2724 -------
2725 2725 An ast.Node corresponding to the node it was called with. Note that it
2726 2726 may also modify the passed object, so don't rely on references to the
2727 2727 original AST.
2728 2728 """
2729 2729 for transformer in self.ast_transformers:
2730 2730 try:
2731 2731 node = transformer.visit(node)
2732 2732 except InputRejected:
2733 2733 # User-supplied AST transformers can reject an input by raising
2734 2734 # an InputRejected. Short-circuit in this case so that we
2735 2735 # don't unregister the transform.
2736 2736 raise
2737 2737 except Exception:
2738 2738 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
2739 2739 self.ast_transformers.remove(transformer)
2740 2740
2741 2741 if self.ast_transformers:
2742 2742 ast.fix_missing_locations(node)
2743 2743 return node
2744 2744
2745 2745
2746 2746 def run_ast_nodes(self, nodelist, cell_name, interactivity='last_expr',
2747 2747 compiler=compile, result=None):
2748 2748 """Run a sequence of AST nodes. The execution mode depends on the
2749 2749 interactivity parameter.
2750 2750
2751 2751 Parameters
2752 2752 ----------
2753 2753 nodelist : list
2754 2754 A sequence of AST nodes to run.
2755 2755 cell_name : str
2756 2756 Will be passed to the compiler as the filename of the cell. Typically
2757 2757 the value returned by ip.compile.cache(cell).
2758 2758 interactivity : str
2759 2759 'all', 'last', 'last_expr' or 'none', specifying which nodes should be
2760 2760 run interactively (displaying output from expressions). 'last_expr'
2761 2761 will run the last node interactively only if it is an expression (i.e.
2762 2762 expressions in loops or other blocks are not displayed. Other values
2763 2763 for this parameter will raise a ValueError.
2764 2764 compiler : callable
2765 2765 A function with the same interface as the built-in compile(), to turn
2766 2766 the AST nodes into code objects. Default is the built-in compile().
2767 2767 result : ExecutionResult, optional
2768 2768 An object to store exceptions that occur during execution.
2769 2769
2770 2770 Returns
2771 2771 -------
2772 2772 True if an exception occurred while running code, False if it finished
2773 2773 running.
2774 2774 """
2775 2775 if not nodelist:
2776 2776 return
2777 2777
2778 2778 if interactivity == 'last_expr':
2779 2779 if isinstance(nodelist[-1], ast.Expr):
2780 2780 interactivity = "last"
2781 2781 else:
2782 2782 interactivity = "none"
2783 2783
2784 2784 if interactivity == 'none':
2785 2785 to_run_exec, to_run_interactive = nodelist, []
2786 2786 elif interactivity == 'last':
2787 2787 to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
2788 2788 elif interactivity == 'all':
2789 2789 to_run_exec, to_run_interactive = [], nodelist
2790 2790 else:
2791 2791 raise ValueError("Interactivity was %r" % interactivity)
2792 2792
2793 2793 try:
2794 2794 for i, node in enumerate(to_run_exec):
2795 2795 mod = ast.Module([node])
2796 2796 code = compiler(mod, cell_name, "exec")
2797 2797 if self.run_code(code, result):
2798 2798 return True
2799 2799
2800 2800 for i, node in enumerate(to_run_interactive):
2801 2801 mod = ast.Interactive([node])
2802 2802 code = compiler(mod, cell_name, "single")
2803 2803 if self.run_code(code, result):
2804 2804 return True
2805 2805
2806 2806 # Flush softspace
2807 2807 if softspace(sys.stdout, 0):
2808 2808 print()
2809 2809
2810 2810 except:
2811 2811 # It's possible to have exceptions raised here, typically by
2812 2812 # compilation of odd code (such as a naked 'return' outside a
2813 2813 # function) that did parse but isn't valid. Typically the exception
2814 2814 # is a SyntaxError, but it's safest just to catch anything and show
2815 2815 # the user a traceback.
2816 2816
2817 2817 # We do only one try/except outside the loop to minimize the impact
2818 2818 # on runtime, and also because if any node in the node list is
2819 2819 # broken, we should stop execution completely.
2820 2820 if result:
2821 2821 result.error_before_exec = sys.exc_info()[1]
2822 2822 self.showtraceback()
2823 2823 return True
2824 2824
2825 2825 return False
2826 2826
2827 2827 def run_code(self, code_obj, result=None):
2828 2828 """Execute a code object.
2829 2829
2830 2830 When an exception occurs, self.showtraceback() is called to display a
2831 2831 traceback.
2832 2832
2833 2833 Parameters
2834 2834 ----------
2835 2835 code_obj : code object
2836 2836 A compiled code object, to be executed
2837 2837 result : ExecutionResult, optional
2838 2838 An object to store exceptions that occur during execution.
2839 2839
2840 2840 Returns
2841 2841 -------
2842 2842 False : successful execution.
2843 2843 True : an error occurred.
2844 2844 """
2845 2845 # Set our own excepthook in case the user code tries to call it
2846 2846 # directly, so that the IPython crash handler doesn't get triggered
2847 2847 old_excepthook, sys.excepthook = sys.excepthook, self.excepthook
2848 2848
2849 2849 # we save the original sys.excepthook in the instance, in case config
2850 2850 # code (such as magics) needs access to it.
2851 2851 self.sys_excepthook = old_excepthook
2852 2852 outflag = 1 # happens in more places, so it's easier as default
2853 2853 try:
2854 2854 try:
2855 2855 self.hooks.pre_run_code_hook()
2856 2856 #rprint('Running code', repr(code_obj)) # dbg
2857 2857 exec(code_obj, self.user_global_ns, self.user_ns)
2858 2858 finally:
2859 2859 # Reset our crash handler in place
2860 2860 sys.excepthook = old_excepthook
2861 2861 except SystemExit as e:
2862 2862 if result is not None:
2863 2863 result.error_in_exec = e
2864 2864 self.showtraceback(exception_only=True)
2865 2865 warn("To exit: use 'exit', 'quit', or Ctrl-D.", stacklevel=1)
2866 2866 except self.custom_exceptions:
2867 2867 etype, value, tb = sys.exc_info()
2868 2868 if result is not None:
2869 2869 result.error_in_exec = value
2870 2870 self.CustomTB(etype, value, tb)
2871 2871 except:
2872 2872 if result is not None:
2873 2873 result.error_in_exec = sys.exc_info()[1]
2874 2874 self.showtraceback()
2875 2875 else:
2876 2876 outflag = 0
2877 2877 return outflag
2878 2878
2879 2879 # For backwards compatibility
2880 2880 runcode = run_code
2881 2881
2882 2882 #-------------------------------------------------------------------------
2883 2883 # Things related to GUI support and pylab
2884 2884 #-------------------------------------------------------------------------
2885 2885
2886 2886 active_eventloop = None
2887 2887
2888 2888 def enable_gui(self, gui=None):
2889 2889 raise NotImplementedError('Implement enable_gui in a subclass')
2890 2890
2891 2891 def enable_matplotlib(self, gui=None):
2892 2892 """Enable interactive matplotlib and inline figure support.
2893 2893
2894 2894 This takes the following steps:
2895 2895
2896 2896 1. select the appropriate eventloop and matplotlib backend
2897 2897 2. set up matplotlib for interactive use with that backend
2898 2898 3. configure formatters for inline figure display
2899 2899 4. enable the selected gui eventloop
2900 2900
2901 2901 Parameters
2902 2902 ----------
2903 2903 gui : optional, string
2904 2904 If given, dictates the choice of matplotlib GUI backend to use
2905 2905 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
2906 2906 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
2907 2907 matplotlib (as dictated by the matplotlib build-time options plus the
2908 2908 user's matplotlibrc configuration file). Note that not all backends
2909 2909 make sense in all contexts, for example a terminal ipython can't
2910 2910 display figures inline.
2911 2911 """
2912 2912 from IPython.core import pylabtools as pt
2913 2913 gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
2914 2914
2915 2915 if gui != 'inline':
2916 2916 # If we have our first gui selection, store it
2917 2917 if self.pylab_gui_select is None:
2918 2918 self.pylab_gui_select = gui
2919 2919 # Otherwise if they are different
2920 2920 elif gui != self.pylab_gui_select:
2921 2921 print ('Warning: Cannot change to a different GUI toolkit: %s.'
2922 2922 ' Using %s instead.' % (gui, self.pylab_gui_select))
2923 2923 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
2924 2924
2925 2925 pt.activate_matplotlib(backend)
2926 2926 pt.configure_inline_support(self, backend)
2927 2927
2928 2928 # Now we must activate the gui pylab wants to use, and fix %run to take
2929 2929 # plot updates into account
2930 2930 self.enable_gui(gui)
2931 2931 self.magics_manager.registry['ExecutionMagics'].default_runner = \
2932 2932 pt.mpl_runner(self.safe_execfile)
2933 2933
2934 2934 return gui, backend
2935 2935
2936 2936 def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
2937 2937 """Activate pylab support at runtime.
2938 2938
2939 2939 This turns on support for matplotlib, preloads into the interactive
2940 2940 namespace all of numpy and pylab, and configures IPython to correctly
2941 2941 interact with the GUI event loop. The GUI backend to be used can be
2942 2942 optionally selected with the optional ``gui`` argument.
2943 2943
2944 2944 This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
2945 2945
2946 2946 Parameters
2947 2947 ----------
2948 2948 gui : optional, string
2949 2949 If given, dictates the choice of matplotlib GUI backend to use
2950 2950 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
2951 2951 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
2952 2952 matplotlib (as dictated by the matplotlib build-time options plus the
2953 2953 user's matplotlibrc configuration file). Note that not all backends
2954 2954 make sense in all contexts, for example a terminal ipython can't
2955 2955 display figures inline.
2956 2956 import_all : optional, bool, default: True
2957 2957 Whether to do `from numpy import *` and `from pylab import *`
2958 2958 in addition to module imports.
2959 2959 welcome_message : deprecated
2960 2960 This argument is ignored, no welcome message will be displayed.
2961 2961 """
2962 2962 from IPython.core.pylabtools import import_pylab
2963 2963
2964 2964 gui, backend = self.enable_matplotlib(gui)
2965 2965
2966 2966 # We want to prevent the loading of pylab to pollute the user's
2967 2967 # namespace as shown by the %who* magics, so we execute the activation
2968 2968 # code in an empty namespace, and we update *both* user_ns and
2969 2969 # user_ns_hidden with this information.
2970 2970 ns = {}
2971 2971 import_pylab(ns, import_all)
2972 2972 # warn about clobbered names
2973 2973 ignored = {"__builtins__"}
2974 2974 both = set(ns).intersection(self.user_ns).difference(ignored)
2975 2975 clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
2976 2976 self.user_ns.update(ns)
2977 2977 self.user_ns_hidden.update(ns)
2978 2978 return gui, backend, clobbered
2979 2979
2980 2980 #-------------------------------------------------------------------------
2981 2981 # Utilities
2982 2982 #-------------------------------------------------------------------------
2983 2983
2984 2984 def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
2985 2985 """Expand python variables in a string.
2986 2986
2987 2987 The depth argument indicates how many frames above the caller should
2988 2988 be walked to look for the local namespace where to expand variables.
2989 2989
2990 2990 The global namespace for expansion is always the user's interactive
2991 2991 namespace.
2992 2992 """
2993 2993 ns = self.user_ns.copy()
2994 2994 try:
2995 2995 frame = sys._getframe(depth+1)
2996 2996 except ValueError:
2997 2997 # This is thrown if there aren't that many frames on the stack,
2998 2998 # e.g. if a script called run_line_magic() directly.
2999 2999 pass
3000 3000 else:
3001 3001 ns.update(frame.f_locals)
3002 3002
3003 3003 try:
3004 3004 # We have to use .vformat() here, because 'self' is a valid and common
3005 3005 # name, and expanding **ns for .format() would make it collide with
3006 3006 # the 'self' argument of the method.
3007 3007 cmd = formatter.vformat(cmd, args=[], kwargs=ns)
3008 3008 except Exception:
3009 3009 # if formatter couldn't format, just let it go untransformed
3010 3010 pass
3011 3011 return cmd
3012 3012
3013 3013 def mktempfile(self, data=None, prefix='ipython_edit_'):
3014 3014 """Make a new tempfile and return its filename.
3015 3015
3016 3016 This makes a call to tempfile.mkstemp (created in a tempfile.mkdtemp),
3017 3017 but it registers the created filename internally so ipython cleans it up
3018 3018 at exit time.
3019 3019
3020 3020 Optional inputs:
3021 3021
3022 3022 - data(None): if data is given, it gets written out to the temp file
3023 3023 immediately, and the file is closed again."""
3024 3024
3025 3025 dirname = tempfile.mkdtemp(prefix=prefix)
3026 3026 self.tempdirs.append(dirname)
3027 3027
3028 3028 handle, filename = tempfile.mkstemp('.py', prefix, dir=dirname)
3029 3029 os.close(handle) # On Windows, there can only be one open handle on a file
3030 3030 self.tempfiles.append(filename)
3031 3031
3032 3032 if data:
3033 3033 tmp_file = open(filename,'w')
3034 3034 tmp_file.write(data)
3035 3035 tmp_file.close()
3036 3036 return filename
3037 3037
3038 3038 @undoc
3039 3039 def write(self,data):
3040 3040 """DEPRECATED: Write a string to the default output"""
3041 3041 warn('InteractiveShell.write() is deprecated, use sys.stdout instead',
3042 3042 DeprecationWarning, stacklevel=2)
3043 3043 sys.stdout.write(data)
3044 3044
3045 3045 @undoc
3046 3046 def write_err(self,data):
3047 3047 """DEPRECATED: Write a string to the default error output"""
3048 3048 warn('InteractiveShell.write_err() is deprecated, use sys.stderr instead',
3049 3049 DeprecationWarning, stacklevel=2)
3050 3050 sys.stderr.write(data)
3051 3051
3052 3052 def ask_yes_no(self, prompt, default=None, interrupt=None):
3053 3053 if self.quiet:
3054 3054 return True
3055 3055 return ask_yes_no(prompt,default,interrupt)
3056 3056
3057 3057 def show_usage(self):
3058 3058 """Show a usage message"""
3059 3059 page.page(IPython.core.usage.interactive_usage)
3060 3060
3061 3061 def extract_input_lines(self, range_str, raw=False):
3062 3062 """Return as a string a set of input history slices.
3063 3063
3064 3064 Parameters
3065 3065 ----------
3066 3066 range_str : string
3067 3067 The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
3068 3068 since this function is for use by magic functions which get their
3069 3069 arguments as strings. The number before the / is the session
3070 3070 number: ~n goes n back from the current session.
3071 3071
3072 3072 raw : bool, optional
3073 3073 By default, the processed input is used. If this is true, the raw
3074 3074 input history is used instead.
3075 3075
3076 3076 Notes
3077 3077 -----
3078 3078
3079 3079 Slices can be described with two notations:
3080 3080
3081 3081 * ``N:M`` -> standard python form, means including items N...(M-1).
3082 3082 * ``N-M`` -> include items N..M (closed endpoint).
3083 3083 """
3084 3084 lines = self.history_manager.get_range_by_str(range_str, raw=raw)
3085 3085 return "\n".join(x for _, _, x in lines)
3086 3086
3087 3087 def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True, search_ns=False):
3088 3088 """Get a code string from history, file, url, or a string or macro.
3089 3089
3090 3090 This is mainly used by magic functions.
3091 3091
3092 3092 Parameters
3093 3093 ----------
3094 3094
3095 3095 target : str
3096 3096
3097 3097 A string specifying code to retrieve. This will be tried respectively
3098 3098 as: ranges of input history (see %history for syntax), url,
3099 3099 corresponding .py file, filename, or an expression evaluating to a
3100 3100 string or Macro in the user namespace.
3101 3101
3102 3102 raw : bool
3103 3103 If true (default), retrieve raw history. Has no effect on the other
3104 3104 retrieval mechanisms.
3105 3105
3106 3106 py_only : bool (default False)
3107 3107 Only try to fetch python code, do not try alternative methods to decode file
3108 3108 if unicode fails.
3109 3109
3110 3110 Returns
3111 3111 -------
3112 3112 A string of code.
3113 3113
3114 3114 ValueError is raised if nothing is found, and TypeError if it evaluates
3115 3115 to an object of another type. In each case, .args[0] is a printable
3116 3116 message.
3117 3117 """
3118 3118 code = self.extract_input_lines(target, raw=raw) # Grab history
3119 3119 if code:
3120 3120 return code
3121 3121 try:
3122 3122 if target.startswith(('http://', 'https://')):
3123 3123 return openpy.read_py_url(target, skip_encoding_cookie=skip_encoding_cookie)
3124 3124 except UnicodeDecodeError:
3125 3125 if not py_only :
3126 3126 # Deferred import
3127 3127 try:
3128 3128 from urllib.request import urlopen # Py3
3129 3129 except ImportError:
3130 3130 from urllib import urlopen
3131 3131 response = urlopen(target)
3132 3132 return response.read().decode('latin1')
3133 3133 raise ValueError(("'%s' seem to be unreadable.") % target)
3134 3134
3135 3135 potential_target = [target]
3136 3136 try :
3137 3137 potential_target.insert(0,get_py_filename(target))
3138 3138 except IOError:
3139 3139 pass
3140 3140
3141 3141 for tgt in potential_target :
3142 3142 if os.path.isfile(tgt): # Read file
3143 3143 try :
3144 3144 return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
3145 3145 except UnicodeDecodeError :
3146 3146 if not py_only :
3147 3147 with io_open(tgt,'r', encoding='latin1') as f :
3148 3148 return f.read()
3149 3149 raise ValueError(("'%s' seem to be unreadable.") % target)
3150 3150 elif os.path.isdir(os.path.expanduser(tgt)):
3151 3151 raise ValueError("'%s' is a directory, not a regular file." % target)
3152 3152
3153 3153 if search_ns:
3154 3154 # Inspect namespace to load object source
3155 3155 object_info = self.object_inspect(target, detail_level=1)
3156 3156 if object_info['found'] and object_info['source']:
3157 3157 return object_info['source']
3158 3158
3159 3159 try: # User namespace
3160 3160 codeobj = eval(target, self.user_ns)
3161 3161 except Exception:
3162 3162 raise ValueError(("'%s' was not found in history, as a file, url, "
3163 3163 "nor in the user namespace.") % target)
3164 3164
3165 3165 if isinstance(codeobj, str):
3166 3166 return codeobj
3167 3167 elif isinstance(codeobj, Macro):
3168 3168 return codeobj.value
3169 3169
3170 3170 raise TypeError("%s is neither a string nor a macro." % target,
3171 3171 codeobj)
3172 3172
3173 3173 #-------------------------------------------------------------------------
3174 3174 # Things related to IPython exiting
3175 3175 #-------------------------------------------------------------------------
3176 3176 def atexit_operations(self):
3177 3177 """This will be executed at the time of exit.
3178 3178
3179 3179 Cleanup operations and saving of persistent data that is done
3180 3180 unconditionally by IPython should be performed here.
3181 3181
3182 3182 For things that may depend on startup flags or platform specifics (such
3183 3183 as having readline or not), register a separate atexit function in the
3184 3184 code that has the appropriate information, rather than trying to
3185 3185 clutter
3186 3186 """
3187 3187 # Close the history session (this stores the end time and line count)
3188 3188 # this must be *before* the tempfile cleanup, in case of temporary
3189 3189 # history db
3190 3190 self.history_manager.end_session()
3191 3191
3192 3192 # Cleanup all tempfiles and folders left around
3193 3193 for tfile in self.tempfiles:
3194 3194 try:
3195 3195 os.unlink(tfile)
3196 3196 except OSError:
3197 3197 pass
3198 3198
3199 3199 for tdir in self.tempdirs:
3200 3200 try:
3201 3201 os.rmdir(tdir)
3202 3202 except OSError:
3203 3203 pass
3204 3204
3205 3205 # Clear all user namespaces to release all references cleanly.
3206 3206 self.reset(new_session=False)
3207 3207
3208 3208 # Run user hooks
3209 3209 self.hooks.shutdown_hook()
3210 3210
3211 3211 def cleanup(self):
3212 3212 self.restore_sys_module_state()
3213 3213
3214 3214
3215 3215 # Overridden in terminal subclass to change prompts
3216 3216 def switch_doctest_mode(self, mode):
3217 3217 pass
3218 3218
3219 3219
3220 3220 class InteractiveShellABC(with_metaclass(abc.ABCMeta, object)):
3221 3221 """An abstract base class for InteractiveShell."""
3222 3222
3223 3223 InteractiveShellABC.register(InteractiveShell)
@@ -1,788 +1,788 b''
1 1 """Implementation of magic functions for interaction with the OS.
2 2
3 3 Note: this module is named 'osm' instead of 'os' to avoid a collision with the
4 4 builtin.
5 5 """
6 6 #-----------------------------------------------------------------------------
7 7 # Copyright (c) 2012 The IPython Development Team.
8 8 #
9 9 # Distributed under the terms of the Modified BSD License.
10 10 #
11 11 # The full license is in the file COPYING.txt, distributed with this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 # Stdlib
19 19 import io
20 20 import os
21 21 import re
22 22 import sys
23 23 from pprint import pformat
24 24
25 25 # Our own packages
26 26 from IPython.core import magic_arguments
27 27 from IPython.core import oinspect
28 28 from IPython.core import page
29 29 from IPython.core.alias import AliasError, Alias
30 30 from IPython.core.error import UsageError
31 31 from IPython.core.magic import (
32 32 Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
33 33 )
34 34 from IPython.testing.skipdoctest import skip_doctest
35 35 from IPython.utils.openpy import source_to_unicode
36 36 from IPython.utils.process import abbrev_cwd
37 37 from IPython.utils import py3compat
38 38 from IPython.utils.terminal import set_term_title
39 39
40 40 #-----------------------------------------------------------------------------
41 41 # Magic implementation classes
42 42 #-----------------------------------------------------------------------------
43 43 @magics_class
44 44 class OSMagics(Magics):
45 45 """Magics to interact with the underlying OS (shell-type functionality).
46 46 """
47 47
48 48 @skip_doctest
49 49 @line_magic
50 50 def alias(self, parameter_s=''):
51 51 """Define an alias for a system command.
52 52
53 53 '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
54 54
55 55 Then, typing 'alias_name params' will execute the system command 'cmd
56 56 params' (from your underlying operating system).
57 57
58 58 Aliases have lower precedence than magic functions and Python normal
59 59 variables, so if 'foo' is both a Python variable and an alias, the
60 60 alias can not be executed until 'del foo' removes the Python variable.
61 61
62 62 You can use the %l specifier in an alias definition to represent the
63 63 whole line when the alias is called. For example::
64 64
65 65 In [2]: alias bracket echo "Input in brackets: <%l>"
66 66 In [3]: bracket hello world
67 67 Input in brackets: <hello world>
68 68
69 69 You can also define aliases with parameters using %s specifiers (one
70 70 per parameter)::
71 71
72 72 In [1]: alias parts echo first %s second %s
73 73 In [2]: %parts A B
74 74 first A second B
75 75 In [3]: %parts A
76 76 Incorrect number of arguments: 2 expected.
77 77 parts is an alias to: 'echo first %s second %s'
78 78
79 79 Note that %l and %s are mutually exclusive. You can only use one or
80 80 the other in your aliases.
81 81
82 82 Aliases expand Python variables just like system calls using ! or !!
83 83 do: all expressions prefixed with '$' get expanded. For details of
84 84 the semantic rules, see PEP-215:
85 85 http://www.python.org/peps/pep-0215.html. This is the library used by
86 86 IPython for variable expansion. If you want to access a true shell
87 87 variable, an extra $ is necessary to prevent its expansion by
88 88 IPython::
89 89
90 90 In [6]: alias show echo
91 91 In [7]: PATH='A Python string'
92 92 In [8]: show $PATH
93 93 A Python string
94 94 In [9]: show $$PATH
95 95 /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
96 96
97 97 You can use the alias facility to acess all of $PATH. See the %rehashx
98 98 function, which automatically creates aliases for the contents of your
99 99 $PATH.
100 100
101 101 If called with no parameters, %alias prints the current alias table."""
102 102
103 103 par = parameter_s.strip()
104 104 if not par:
105 105 aliases = sorted(self.shell.alias_manager.aliases)
106 106 # stored = self.shell.db.get('stored_aliases', {} )
107 107 # for k, v in stored:
108 108 # atab.append(k, v[0])
109 109
110 110 print("Total number of aliases:", len(aliases))
111 111 sys.stdout.flush()
112 112 return aliases
113 113
114 114 # Now try to define a new one
115 115 try:
116 116 alias,cmd = par.split(None, 1)
117 117 except TypeError:
118 118 print(oinspect.getdoc(self.alias))
119 119 return
120 120
121 121 try:
122 122 self.shell.alias_manager.define_alias(alias, cmd)
123 123 except AliasError as e:
124 124 print(e)
125 125 # end magic_alias
126 126
127 127 @line_magic
128 128 def unalias(self, parameter_s=''):
129 129 """Remove an alias"""
130 130
131 131 aname = parameter_s.strip()
132 132 try:
133 133 self.shell.alias_manager.undefine_alias(aname)
134 134 except ValueError as e:
135 135 print(e)
136 136 return
137 137
138 138 stored = self.shell.db.get('stored_aliases', {} )
139 139 if aname in stored:
140 140 print("Removing %stored alias",aname)
141 141 del stored[aname]
142 142 self.shell.db['stored_aliases'] = stored
143 143
144 144 @line_magic
145 145 def rehashx(self, parameter_s=''):
146 146 """Update the alias table with all executable files in $PATH.
147 147
148 148 rehashx explicitly checks that every entry in $PATH is a file
149 149 with execute access (os.X_OK).
150 150
151 151 Under Windows, it checks executability as a match against a
152 152 '|'-separated string of extensions, stored in the IPython config
153 153 variable win_exec_ext. This defaults to 'exe|com|bat'.
154 154
155 155 This function also resets the root module cache of module completer,
156 156 used on slow filesystems.
157 157 """
158 158 from IPython.core.alias import InvalidAliasError
159 159
160 160 # for the benefit of module completer in ipy_completers.py
161 161 del self.shell.db['rootmodules_cache']
162 162
163 163 path = [os.path.abspath(os.path.expanduser(p)) for p in
164 164 os.environ.get('PATH','').split(os.pathsep)]
165 165
166 166 syscmdlist = []
167 167 # Now define isexec in a cross platform manner.
168 168 if os.name == 'posix':
169 169 isexec = lambda fname:os.path.isfile(fname) and \
170 170 os.access(fname,os.X_OK)
171 171 else:
172 172 try:
173 173 winext = os.environ['pathext'].replace(';','|').replace('.','')
174 174 except KeyError:
175 175 winext = 'exe|com|bat|py'
176 176 if 'py' not in winext:
177 177 winext += '|py'
178 178 execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
179 179 isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
180 savedir = py3compat.getcwd()
180 savedir = os.getcwd()
181 181
182 182 # Now walk the paths looking for executables to alias.
183 183 try:
184 184 # write the whole loop for posix/Windows so we don't have an if in
185 185 # the innermost part
186 186 if os.name == 'posix':
187 187 for pdir in path:
188 188 try:
189 189 os.chdir(pdir)
190 190 dirlist = os.listdir(pdir)
191 191 except OSError:
192 192 continue
193 193 for ff in dirlist:
194 194 if isexec(ff):
195 195 try:
196 196 # Removes dots from the name since ipython
197 197 # will assume names with dots to be python.
198 198 if not self.shell.alias_manager.is_alias(ff):
199 199 self.shell.alias_manager.define_alias(
200 200 ff.replace('.',''), ff)
201 201 except InvalidAliasError:
202 202 pass
203 203 else:
204 204 syscmdlist.append(ff)
205 205 else:
206 206 no_alias = Alias.blacklist
207 207 for pdir in path:
208 208 try:
209 209 os.chdir(pdir)
210 210 dirlist = os.listdir(pdir)
211 211 except OSError:
212 212 continue
213 213 for ff in dirlist:
214 214 base, ext = os.path.splitext(ff)
215 215 if isexec(ff) and base.lower() not in no_alias:
216 216 if ext.lower() == '.exe':
217 217 ff = base
218 218 try:
219 219 # Removes dots from the name since ipython
220 220 # will assume names with dots to be python.
221 221 self.shell.alias_manager.define_alias(
222 222 base.lower().replace('.',''), ff)
223 223 except InvalidAliasError:
224 224 pass
225 225 syscmdlist.append(ff)
226 226 self.shell.db['syscmdlist'] = syscmdlist
227 227 finally:
228 228 os.chdir(savedir)
229 229
230 230 @skip_doctest
231 231 @line_magic
232 232 def pwd(self, parameter_s=''):
233 233 """Return the current working directory path.
234 234
235 235 Examples
236 236 --------
237 237 ::
238 238
239 239 In [9]: pwd
240 240 Out[9]: '/home/tsuser/sprint/ipython'
241 241 """
242 return py3compat.getcwd()
242 return os.getcwd()
243 243
244 244 @skip_doctest
245 245 @line_magic
246 246 def cd(self, parameter_s=''):
247 247 """Change the current working directory.
248 248
249 249 This command automatically maintains an internal list of directories
250 250 you visit during your IPython session, in the variable _dh. The
251 251 command %dhist shows this history nicely formatted. You can also
252 252 do 'cd -<tab>' to see directory history conveniently.
253 253
254 254 Usage:
255 255
256 256 cd 'dir': changes to directory 'dir'.
257 257
258 258 cd -: changes to the last visited directory.
259 259
260 260 cd -<n>: changes to the n-th directory in the directory history.
261 261
262 262 cd --foo: change to directory that matches 'foo' in history
263 263
264 264 cd -b <bookmark_name>: jump to a bookmark set by %bookmark
265 265 (note: cd <bookmark_name> is enough if there is no
266 266 directory <bookmark_name>, but a bookmark with the name exists.)
267 267 'cd -b <tab>' allows you to tab-complete bookmark names.
268 268
269 269 Options:
270 270
271 271 -q: quiet. Do not print the working directory after the cd command is
272 272 executed. By default IPython's cd command does print this directory,
273 273 since the default prompts do not display path information.
274 274
275 275 Note that !cd doesn't work for this purpose because the shell where
276 276 !command runs is immediately discarded after executing 'command'.
277 277
278 278 Examples
279 279 --------
280 280 ::
281 281
282 282 In [10]: cd parent/child
283 283 /home/tsuser/parent/child
284 284 """
285 285
286 oldcwd = py3compat.getcwd()
286 oldcwd = os.getcwd()
287 287 numcd = re.match(r'(-)(\d+)$',parameter_s)
288 288 # jump in directory history by number
289 289 if numcd:
290 290 nn = int(numcd.group(2))
291 291 try:
292 292 ps = self.shell.user_ns['_dh'][nn]
293 293 except IndexError:
294 294 print('The requested directory does not exist in history.')
295 295 return
296 296 else:
297 297 opts = {}
298 298 elif parameter_s.startswith('--'):
299 299 ps = None
300 300 fallback = None
301 301 pat = parameter_s[2:]
302 302 dh = self.shell.user_ns['_dh']
303 303 # first search only by basename (last component)
304 304 for ent in reversed(dh):
305 305 if pat in os.path.basename(ent) and os.path.isdir(ent):
306 306 ps = ent
307 307 break
308 308
309 309 if fallback is None and pat in ent and os.path.isdir(ent):
310 310 fallback = ent
311 311
312 312 # if we have no last part match, pick the first full path match
313 313 if ps is None:
314 314 ps = fallback
315 315
316 316 if ps is None:
317 317 print("No matching entry in directory history")
318 318 return
319 319 else:
320 320 opts = {}
321 321
322 322
323 323 else:
324 324 opts, ps = self.parse_options(parameter_s, 'qb', mode='string')
325 325 # jump to previous
326 326 if ps == '-':
327 327 try:
328 328 ps = self.shell.user_ns['_dh'][-2]
329 329 except IndexError:
330 330 raise UsageError('%cd -: No previous directory to change to.')
331 331 # jump to bookmark if needed
332 332 else:
333 333 if not os.path.isdir(ps) or 'b' in opts:
334 334 bkms = self.shell.db.get('bookmarks', {})
335 335
336 336 if ps in bkms:
337 337 target = bkms[ps]
338 338 print('(bookmark:%s) -> %s' % (ps, target))
339 339 ps = target
340 340 else:
341 341 if 'b' in opts:
342 342 raise UsageError("Bookmark '%s' not found. "
343 343 "Use '%%bookmark -l' to see your bookmarks." % ps)
344 344
345 345 # at this point ps should point to the target dir
346 346 if ps:
347 347 try:
348 348 os.chdir(os.path.expanduser(ps))
349 349 if hasattr(self.shell, 'term_title') and self.shell.term_title:
350 350 set_term_title('IPython: ' + abbrev_cwd())
351 351 except OSError:
352 352 print(sys.exc_info()[1])
353 353 else:
354 cwd = py3compat.getcwd()
354 cwd = os.getcwd()
355 355 dhist = self.shell.user_ns['_dh']
356 356 if oldcwd != cwd:
357 357 dhist.append(cwd)
358 358 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
359 359
360 360 else:
361 361 os.chdir(self.shell.home_dir)
362 362 if hasattr(self.shell, 'term_title') and self.shell.term_title:
363 363 set_term_title('IPython: ' + '~')
364 cwd = py3compat.getcwd()
364 cwd = os.getcwd()
365 365 dhist = self.shell.user_ns['_dh']
366 366
367 367 if oldcwd != cwd:
368 368 dhist.append(cwd)
369 369 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
370 370 if not 'q' in opts and self.shell.user_ns['_dh']:
371 371 print(self.shell.user_ns['_dh'][-1])
372 372
373 373 @line_magic
374 374 def env(self, parameter_s=''):
375 375 """Get, set, or list environment variables.
376 376
377 377 Usage:\\
378 378
379 379 %env: lists all environment variables/values
380 380 %env var: get value for var
381 381 %env var val: set value for var
382 382 %env var=val: set value for var
383 383 %env var=$val: set value for var, using python expansion if possible
384 384 """
385 385 if parameter_s.strip():
386 386 split = '=' if '=' in parameter_s else ' '
387 387 bits = parameter_s.split(split)
388 388 if len(bits) == 1:
389 389 key = parameter_s.strip()
390 390 if key in os.environ:
391 391 return os.environ[key]
392 392 else:
393 393 err = "Environment does not have key: {0}".format(key)
394 394 raise UsageError(err)
395 395 if len(bits) > 1:
396 396 return self.set_env(parameter_s)
397 397 return dict(os.environ)
398 398
399 399 @line_magic
400 400 def set_env(self, parameter_s):
401 401 """Set environment variables. Assumptions are that either "val" is a
402 402 name in the user namespace, or val is something that evaluates to a
403 403 string.
404 404
405 405 Usage:\\
406 406 %set_env var val: set value for var
407 407 %set_env var=val: set value for var
408 408 %set_env var=$val: set value for var, using python expansion if possible
409 409 """
410 410 split = '=' if '=' in parameter_s else ' '
411 411 bits = parameter_s.split(split, 1)
412 412 if not parameter_s.strip() or len(bits)<2:
413 413 raise UsageError("usage is 'set_env var=val'")
414 414 var = bits[0].strip()
415 415 val = bits[1].strip()
416 416 if re.match(r'.*\s.*', var):
417 417 # an environment variable with whitespace is almost certainly
418 418 # not what the user intended. what's more likely is the wrong
419 419 # split was chosen, ie for "set_env cmd_args A=B", we chose
420 420 # '=' for the split and should have chosen ' '. to get around
421 421 # this, users should just assign directly to os.environ or use
422 422 # standard magic {var} expansion.
423 423 err = "refusing to set env var with whitespace: '{0}'"
424 424 err = err.format(val)
425 425 raise UsageError(err)
426 426 os.environ[py3compat.cast_bytes_py2(var)] = py3compat.cast_bytes_py2(val)
427 427 print('env: {0}={1}'.format(var,val))
428 428
429 429 @line_magic
430 430 def pushd(self, parameter_s=''):
431 431 """Place the current dir on stack and change directory.
432 432
433 433 Usage:\\
434 434 %pushd ['dirname']
435 435 """
436 436
437 437 dir_s = self.shell.dir_stack
438 438 tgt = os.path.expanduser(parameter_s)
439 cwd = py3compat.getcwd().replace(self.shell.home_dir,'~')
439 cwd = os.getcwd().replace(self.shell.home_dir,'~')
440 440 if tgt:
441 441 self.cd(parameter_s)
442 442 dir_s.insert(0,cwd)
443 443 return self.shell.magic('dirs')
444 444
445 445 @line_magic
446 446 def popd(self, parameter_s=''):
447 447 """Change to directory popped off the top of the stack.
448 448 """
449 449 if not self.shell.dir_stack:
450 450 raise UsageError("%popd on empty stack")
451 451 top = self.shell.dir_stack.pop(0)
452 452 self.cd(top)
453 453 print("popd ->",top)
454 454
455 455 @line_magic
456 456 def dirs(self, parameter_s=''):
457 457 """Return the current directory stack."""
458 458
459 459 return self.shell.dir_stack
460 460
461 461 @line_magic
462 462 def dhist(self, parameter_s=''):
463 463 """Print your history of visited directories.
464 464
465 465 %dhist -> print full history\\
466 466 %dhist n -> print last n entries only\\
467 467 %dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
468 468
469 469 This history is automatically maintained by the %cd command, and
470 470 always available as the global list variable _dh. You can use %cd -<n>
471 471 to go to directory number <n>.
472 472
473 473 Note that most of time, you should view directory history by entering
474 474 cd -<TAB>.
475 475
476 476 """
477 477
478 478 dh = self.shell.user_ns['_dh']
479 479 if parameter_s:
480 480 try:
481 481 args = map(int,parameter_s.split())
482 482 except:
483 483 self.arg_err(self.dhist)
484 484 return
485 485 if len(args) == 1:
486 486 ini,fin = max(len(dh)-(args[0]),0),len(dh)
487 487 elif len(args) == 2:
488 488 ini,fin = args
489 489 fin = min(fin, len(dh))
490 490 else:
491 491 self.arg_err(self.dhist)
492 492 return
493 493 else:
494 494 ini,fin = 0,len(dh)
495 495 print('Directory history (kept in _dh)')
496 496 for i in range(ini, fin):
497 497 print("%d: %s" % (i, dh[i]))
498 498
499 499 @skip_doctest
500 500 @line_magic
501 501 def sc(self, parameter_s=''):
502 502 """Shell capture - run shell command and capture output (DEPRECATED use !).
503 503
504 504 DEPRECATED. Suboptimal, retained for backwards compatibility.
505 505
506 506 You should use the form 'var = !command' instead. Example:
507 507
508 508 "%sc -l myfiles = ls ~" should now be written as
509 509
510 510 "myfiles = !ls ~"
511 511
512 512 myfiles.s, myfiles.l and myfiles.n still apply as documented
513 513 below.
514 514
515 515 --
516 516 %sc [options] varname=command
517 517
518 518 IPython will run the given command using commands.getoutput(), and
519 519 will then update the user's interactive namespace with a variable
520 520 called varname, containing the value of the call. Your command can
521 521 contain shell wildcards, pipes, etc.
522 522
523 523 The '=' sign in the syntax is mandatory, and the variable name you
524 524 supply must follow Python's standard conventions for valid names.
525 525
526 526 (A special format without variable name exists for internal use)
527 527
528 528 Options:
529 529
530 530 -l: list output. Split the output on newlines into a list before
531 531 assigning it to the given variable. By default the output is stored
532 532 as a single string.
533 533
534 534 -v: verbose. Print the contents of the variable.
535 535
536 536 In most cases you should not need to split as a list, because the
537 537 returned value is a special type of string which can automatically
538 538 provide its contents either as a list (split on newlines) or as a
539 539 space-separated string. These are convenient, respectively, either
540 540 for sequential processing or to be passed to a shell command.
541 541
542 542 For example::
543 543
544 544 # Capture into variable a
545 545 In [1]: sc a=ls *py
546 546
547 547 # a is a string with embedded newlines
548 548 In [2]: a
549 549 Out[2]: 'setup.py\\nwin32_manual_post_install.py'
550 550
551 551 # which can be seen as a list:
552 552 In [3]: a.l
553 553 Out[3]: ['setup.py', 'win32_manual_post_install.py']
554 554
555 555 # or as a whitespace-separated string:
556 556 In [4]: a.s
557 557 Out[4]: 'setup.py win32_manual_post_install.py'
558 558
559 559 # a.s is useful to pass as a single command line:
560 560 In [5]: !wc -l $a.s
561 561 146 setup.py
562 562 130 win32_manual_post_install.py
563 563 276 total
564 564
565 565 # while the list form is useful to loop over:
566 566 In [6]: for f in a.l:
567 567 ...: !wc -l $f
568 568 ...:
569 569 146 setup.py
570 570 130 win32_manual_post_install.py
571 571
572 572 Similarly, the lists returned by the -l option are also special, in
573 573 the sense that you can equally invoke the .s attribute on them to
574 574 automatically get a whitespace-separated string from their contents::
575 575
576 576 In [7]: sc -l b=ls *py
577 577
578 578 In [8]: b
579 579 Out[8]: ['setup.py', 'win32_manual_post_install.py']
580 580
581 581 In [9]: b.s
582 582 Out[9]: 'setup.py win32_manual_post_install.py'
583 583
584 584 In summary, both the lists and strings used for output capture have
585 585 the following special attributes::
586 586
587 587 .l (or .list) : value as list.
588 588 .n (or .nlstr): value as newline-separated string.
589 589 .s (or .spstr): value as space-separated string.
590 590 """
591 591
592 592 opts,args = self.parse_options(parameter_s, 'lv')
593 593 # Try to get a variable name and command to run
594 594 try:
595 595 # the variable name must be obtained from the parse_options
596 596 # output, which uses shlex.split to strip options out.
597 597 var,_ = args.split('=', 1)
598 598 var = var.strip()
599 599 # But the command has to be extracted from the original input
600 600 # parameter_s, not on what parse_options returns, to avoid the
601 601 # quote stripping which shlex.split performs on it.
602 602 _,cmd = parameter_s.split('=', 1)
603 603 except ValueError:
604 604 var,cmd = '',''
605 605 # If all looks ok, proceed
606 606 split = 'l' in opts
607 607 out = self.shell.getoutput(cmd, split=split)
608 608 if 'v' in opts:
609 609 print('%s ==\n%s' % (var, pformat(out)))
610 610 if var:
611 611 self.shell.user_ns.update({var:out})
612 612 else:
613 613 return out
614 614
615 615 @line_cell_magic
616 616 def sx(self, line='', cell=None):
617 617 """Shell execute - run shell command and capture output (!! is short-hand).
618 618
619 619 %sx command
620 620
621 621 IPython will run the given command using commands.getoutput(), and
622 622 return the result formatted as a list (split on '\\n'). Since the
623 623 output is _returned_, it will be stored in ipython's regular output
624 624 cache Out[N] and in the '_N' automatic variables.
625 625
626 626 Notes:
627 627
628 628 1) If an input line begins with '!!', then %sx is automatically
629 629 invoked. That is, while::
630 630
631 631 !ls
632 632
633 633 causes ipython to simply issue system('ls'), typing::
634 634
635 635 !!ls
636 636
637 637 is a shorthand equivalent to::
638 638
639 639 %sx ls
640 640
641 641 2) %sx differs from %sc in that %sx automatically splits into a list,
642 642 like '%sc -l'. The reason for this is to make it as easy as possible
643 643 to process line-oriented shell output via further python commands.
644 644 %sc is meant to provide much finer control, but requires more
645 645 typing.
646 646
647 647 3) Just like %sc -l, this is a list with special attributes:
648 648 ::
649 649
650 650 .l (or .list) : value as list.
651 651 .n (or .nlstr): value as newline-separated string.
652 652 .s (or .spstr): value as whitespace-separated string.
653 653
654 654 This is very useful when trying to use such lists as arguments to
655 655 system commands."""
656 656
657 657 if cell is None:
658 658 # line magic
659 659 return self.shell.getoutput(line)
660 660 else:
661 661 opts,args = self.parse_options(line, '', 'out=')
662 662 output = self.shell.getoutput(cell)
663 663 out_name = opts.get('out', opts.get('o'))
664 664 if out_name:
665 665 self.shell.user_ns[out_name] = output
666 666 else:
667 667 return output
668 668
669 669 system = line_cell_magic('system')(sx)
670 670 bang = cell_magic('!')(sx)
671 671
672 672 @line_magic
673 673 def bookmark(self, parameter_s=''):
674 674 """Manage IPython's bookmark system.
675 675
676 676 %bookmark <name> - set bookmark to current dir
677 677 %bookmark <name> <dir> - set bookmark to <dir>
678 678 %bookmark -l - list all bookmarks
679 679 %bookmark -d <name> - remove bookmark
680 680 %bookmark -r - remove all bookmarks
681 681
682 682 You can later on access a bookmarked folder with::
683 683
684 684 %cd -b <name>
685 685
686 686 or simply '%cd <name>' if there is no directory called <name> AND
687 687 there is such a bookmark defined.
688 688
689 689 Your bookmarks persist through IPython sessions, but they are
690 690 associated with each profile."""
691 691
692 692 opts,args = self.parse_options(parameter_s,'drl',mode='list')
693 693 if len(args) > 2:
694 694 raise UsageError("%bookmark: too many arguments")
695 695
696 696 bkms = self.shell.db.get('bookmarks',{})
697 697
698 698 if 'd' in opts:
699 699 try:
700 700 todel = args[0]
701 701 except IndexError:
702 702 raise UsageError(
703 703 "%bookmark -d: must provide a bookmark to delete")
704 704 else:
705 705 try:
706 706 del bkms[todel]
707 707 except KeyError:
708 708 raise UsageError(
709 709 "%%bookmark -d: Can't delete bookmark '%s'" % todel)
710 710
711 711 elif 'r' in opts:
712 712 bkms = {}
713 713 elif 'l' in opts:
714 714 bks = sorted(bkms)
715 715 if bks:
716 716 size = max(map(len, bks))
717 717 else:
718 718 size = 0
719 719 fmt = '%-'+str(size)+'s -> %s'
720 720 print('Current bookmarks:')
721 721 for bk in bks:
722 722 print(fmt % (bk, bkms[bk]))
723 723 else:
724 724 if not args:
725 725 raise UsageError("%bookmark: You must specify the bookmark name")
726 726 elif len(args)==1:
727 bkms[args[0]] = py3compat.getcwd()
727 bkms[args[0]] = os.getcwd()
728 728 elif len(args)==2:
729 729 bkms[args[0]] = args[1]
730 730 self.shell.db['bookmarks'] = bkms
731 731
732 732 @line_magic
733 733 def pycat(self, parameter_s=''):
734 734 """Show a syntax-highlighted file through a pager.
735 735
736 736 This magic is similar to the cat utility, but it will assume the file
737 737 to be Python source and will show it with syntax highlighting.
738 738
739 739 This magic command can either take a local filename, an url,
740 740 an history range (see %history) or a macro as argument ::
741 741
742 742 %pycat myscript.py
743 743 %pycat 7-27
744 744 %pycat myMacro
745 745 %pycat http://www.example.com/myscript.py
746 746 """
747 747 if not parameter_s:
748 748 raise UsageError('Missing filename, URL, input history range, '
749 749 'or macro.')
750 750
751 751 try :
752 752 cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
753 753 except (ValueError, IOError):
754 754 print("Error: no such file, variable, URL, history range or macro")
755 755 return
756 756
757 757 page.page(self.shell.pycolorize(source_to_unicode(cont)))
758 758
759 759 @magic_arguments.magic_arguments()
760 760 @magic_arguments.argument(
761 761 '-a', '--append', action='store_true', default=False,
762 762 help='Append contents of the cell to an existing file. '
763 763 'The file will be created if it does not exist.'
764 764 )
765 765 @magic_arguments.argument(
766 766 'filename', type=str,
767 767 help='file to write'
768 768 )
769 769 @cell_magic
770 770 def writefile(self, line, cell):
771 771 """Write the contents of the cell to a file.
772 772
773 773 The file will be overwritten unless the -a (--append) flag is specified.
774 774 """
775 775 args = magic_arguments.parse_argstring(self.writefile, line)
776 776 filename = os.path.expanduser(args.filename)
777 777
778 778 if os.path.exists(filename):
779 779 if args.append:
780 780 print("Appending to %s" % filename)
781 781 else:
782 782 print("Overwriting %s" % filename)
783 783 else:
784 784 print("Writing %s" % filename)
785 785
786 786 mode = 'a' if args.append else 'w'
787 787 with io.open(filename, mode, encoding='utf-8') as f:
788 788 f.write(cell)
@@ -1,313 +1,313 b''
1 1 # encoding: utf-8
2 2 """
3 3 An application for managing IPython profiles.
4 4
5 5 To be invoked as the `ipython profile` subcommand.
6 6
7 7 Authors:
8 8
9 9 * Min RK
10 10
11 11 """
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Copyright (C) 2008 The IPython Development Team
15 15 #
16 16 # Distributed under the terms of the BSD License. The full license is in
17 17 # the file COPYING, distributed as part of this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Imports
22 22 #-----------------------------------------------------------------------------
23 23
24 24 import os
25 25
26 26 from traitlets.config.application import Application
27 27 from IPython.core.application import (
28 28 BaseIPythonApplication, base_flags
29 29 )
30 30 from IPython.core.profiledir import ProfileDir
31 31 from IPython.utils.importstring import import_item
32 32 from IPython.paths import get_ipython_dir, get_ipython_package_dir
33 33 from IPython.utils import py3compat
34 34 from traitlets import Unicode, Bool, Dict, observe
35 35
36 36 #-----------------------------------------------------------------------------
37 37 # Constants
38 38 #-----------------------------------------------------------------------------
39 39
40 40 create_help = """Create an IPython profile by name
41 41
42 42 Create an ipython profile directory by its name or
43 43 profile directory path. Profile directories contain
44 44 configuration, log and security related files and are named
45 45 using the convention 'profile_<name>'. By default they are
46 46 located in your ipython directory. Once created, you will
47 47 can edit the configuration files in the profile
48 48 directory to configure IPython. Most users will create a
49 49 profile directory by name,
50 50 `ipython profile create myprofile`, which will put the directory
51 51 in `<ipython_dir>/profile_myprofile`.
52 52 """
53 53 list_help = """List available IPython profiles
54 54
55 55 List all available profiles, by profile location, that can
56 56 be found in the current working directly or in the ipython
57 57 directory. Profile directories are named using the convention
58 58 'profile_<profile>'.
59 59 """
60 60 profile_help = """Manage IPython profiles
61 61
62 62 Profile directories contain
63 63 configuration, log and security related files and are named
64 64 using the convention 'profile_<name>'. By default they are
65 65 located in your ipython directory. You can create profiles
66 66 with `ipython profile create <name>`, or see the profiles you
67 67 already have with `ipython profile list`
68 68
69 69 To get started configuring IPython, simply do:
70 70
71 71 $> ipython profile create
72 72
73 73 and IPython will create the default profile in <ipython_dir>/profile_default,
74 74 where you can edit ipython_config.py to start configuring IPython.
75 75
76 76 """
77 77
78 78 _list_examples = "ipython profile list # list all profiles"
79 79
80 80 _create_examples = """
81 81 ipython profile create foo # create profile foo w/ default config files
82 82 ipython profile create foo --reset # restage default config files over current
83 83 ipython profile create foo --parallel # also stage parallel config files
84 84 """
85 85
86 86 _main_examples = """
87 87 ipython profile create -h # show the help string for the create subcommand
88 88 ipython profile list -h # show the help string for the list subcommand
89 89
90 90 ipython locate profile foo # print the path to the directory for profile 'foo'
91 91 """
92 92
93 93 #-----------------------------------------------------------------------------
94 94 # Profile Application Class (for `ipython profile` subcommand)
95 95 #-----------------------------------------------------------------------------
96 96
97 97
98 98 def list_profiles_in(path):
99 99 """list profiles in a given root directory"""
100 100 files = os.listdir(path)
101 101 profiles = []
102 102 for f in files:
103 103 try:
104 104 full_path = os.path.join(path, f)
105 105 except UnicodeError:
106 106 continue
107 107 if os.path.isdir(full_path) and f.startswith('profile_'):
108 108 profiles.append(f.split('_',1)[-1])
109 109 return profiles
110 110
111 111
112 112 def list_bundled_profiles():
113 113 """list profiles that are bundled with IPython."""
114 114 path = os.path.join(get_ipython_package_dir(), u'core', u'profile')
115 115 files = os.listdir(path)
116 116 profiles = []
117 117 for profile in files:
118 118 full_path = os.path.join(path, profile)
119 119 if os.path.isdir(full_path) and profile != "__pycache__":
120 120 profiles.append(profile)
121 121 return profiles
122 122
123 123
124 124 class ProfileLocate(BaseIPythonApplication):
125 125 description = """print the path to an IPython profile dir"""
126 126
127 127 def parse_command_line(self, argv=None):
128 128 super(ProfileLocate, self).parse_command_line(argv)
129 129 if self.extra_args:
130 130 self.profile = self.extra_args[0]
131 131
132 132 def start(self):
133 133 print(self.profile_dir.location)
134 134
135 135
136 136 class ProfileList(Application):
137 137 name = u'ipython-profile'
138 138 description = list_help
139 139 examples = _list_examples
140 140
141 141 aliases = Dict({
142 142 'ipython-dir' : 'ProfileList.ipython_dir',
143 143 'log-level' : 'Application.log_level',
144 144 })
145 145 flags = Dict(dict(
146 146 debug = ({'Application' : {'log_level' : 0}},
147 147 "Set Application.log_level to 0, maximizing log output."
148 148 )
149 149 ))
150 150
151 151 ipython_dir = Unicode(get_ipython_dir(),
152 152 help="""
153 153 The name of the IPython directory. This directory is used for logging
154 154 configuration (through profiles), history storage, etc. The default
155 155 is usually $HOME/.ipython. This options can also be specified through
156 156 the environment variable IPYTHONDIR.
157 157 """
158 158 ).tag(config=True)
159 159
160 160
161 161 def _print_profiles(self, profiles):
162 162 """print list of profiles, indented."""
163 163 for profile in profiles:
164 164 print(' %s' % profile)
165 165
166 166 def list_profile_dirs(self):
167 167 profiles = list_bundled_profiles()
168 168 if profiles:
169 169 print()
170 170 print("Available profiles in IPython:")
171 171 self._print_profiles(profiles)
172 172 print()
173 173 print(" The first request for a bundled profile will copy it")
174 174 print(" into your IPython directory (%s)," % self.ipython_dir)
175 175 print(" where you can customize it.")
176 176
177 177 profiles = list_profiles_in(self.ipython_dir)
178 178 if profiles:
179 179 print()
180 180 print("Available profiles in %s:" % self.ipython_dir)
181 181 self._print_profiles(profiles)
182 182
183 profiles = list_profiles_in(py3compat.getcwd())
183 profiles = list_profiles_in(os.getcwd())
184 184 if profiles:
185 185 print()
186 print("Available profiles in current directory (%s):" % py3compat.getcwd())
186 print("Available profiles in current directory (%s):" % os.getcwd())
187 187 self._print_profiles(profiles)
188 188
189 189 print()
190 190 print("To use any of the above profiles, start IPython with:")
191 191 print(" ipython --profile=<name>")
192 192 print()
193 193
194 194 def start(self):
195 195 self.list_profile_dirs()
196 196
197 197
198 198 create_flags = {}
199 199 create_flags.update(base_flags)
200 200 # don't include '--init' flag, which implies running profile create in other apps
201 201 create_flags.pop('init')
202 202 create_flags['reset'] = ({'ProfileCreate': {'overwrite' : True}},
203 203 "reset config files in this profile to the defaults.")
204 204 create_flags['parallel'] = ({'ProfileCreate': {'parallel' : True}},
205 205 "Include the config files for parallel "
206 206 "computing apps (ipengine, ipcontroller, etc.)")
207 207
208 208
209 209 class ProfileCreate(BaseIPythonApplication):
210 210 name = u'ipython-profile'
211 211 description = create_help
212 212 examples = _create_examples
213 213 auto_create = Bool(True)
214 214 def _log_format_default(self):
215 215 return "[%(name)s] %(message)s"
216 216
217 217 def _copy_config_files_default(self):
218 218 return True
219 219
220 220 parallel = Bool(False,
221 221 help="whether to include parallel computing config files"
222 222 ).tag(config=True)
223 223
224 224 @observe('parallel')
225 225 def _parallel_changed(self, change):
226 226 parallel_files = [ 'ipcontroller_config.py',
227 227 'ipengine_config.py',
228 228 'ipcluster_config.py'
229 229 ]
230 230 if change['new']:
231 231 for cf in parallel_files:
232 232 self.config_files.append(cf)
233 233 else:
234 234 for cf in parallel_files:
235 235 if cf in self.config_files:
236 236 self.config_files.remove(cf)
237 237
238 238 def parse_command_line(self, argv):
239 239 super(ProfileCreate, self).parse_command_line(argv)
240 240 # accept positional arg as profile name
241 241 if self.extra_args:
242 242 self.profile = self.extra_args[0]
243 243
244 244 flags = Dict(create_flags)
245 245
246 246 classes = [ProfileDir]
247 247
248 248 def _import_app(self, app_path):
249 249 """import an app class"""
250 250 app = None
251 251 name = app_path.rsplit('.', 1)[-1]
252 252 try:
253 253 app = import_item(app_path)
254 254 except ImportError:
255 255 self.log.info("Couldn't import %s, config file will be excluded", name)
256 256 except Exception:
257 257 self.log.warning('Unexpected error importing %s', name, exc_info=True)
258 258 return app
259 259
260 260 def init_config_files(self):
261 261 super(ProfileCreate, self).init_config_files()
262 262 # use local imports, since these classes may import from here
263 263 from IPython.terminal.ipapp import TerminalIPythonApp
264 264 apps = [TerminalIPythonApp]
265 265 for app_path in (
266 266 'ipykernel.kernelapp.IPKernelApp',
267 267 ):
268 268 app = self._import_app(app_path)
269 269 if app is not None:
270 270 apps.append(app)
271 271 if self.parallel:
272 272 from ipyparallel.apps.ipcontrollerapp import IPControllerApp
273 273 from ipyparallel.apps.ipengineapp import IPEngineApp
274 274 from ipyparallel.apps.ipclusterapp import IPClusterStart
275 275 apps.extend([
276 276 IPControllerApp,
277 277 IPEngineApp,
278 278 IPClusterStart,
279 279 ])
280 280 for App in apps:
281 281 app = App()
282 282 app.config.update(self.config)
283 283 app.log = self.log
284 284 app.overwrite = self.overwrite
285 285 app.copy_config_files=True
286 286 app.ipython_dir=self.ipython_dir
287 287 app.profile_dir=self.profile_dir
288 288 app.init_config_files()
289 289
290 290 def stage_default_config_file(self):
291 291 pass
292 292
293 293
294 294 class ProfileApp(Application):
295 295 name = u'ipython profile'
296 296 description = profile_help
297 297 examples = _main_examples
298 298
299 299 subcommands = Dict(dict(
300 300 create = (ProfileCreate, ProfileCreate.description.splitlines()[0]),
301 301 list = (ProfileList, ProfileList.description.splitlines()[0]),
302 302 locate = (ProfileLocate, ProfileLocate.description.splitlines()[0]),
303 303 ))
304 304
305 305 def start(self):
306 306 if self.subapp is None:
307 307 print("No subcommand specified. Must specify one of: %s"%(self.subcommands.keys()))
308 308 print()
309 309 self.print_description()
310 310 self.print_subcommands()
311 311 self.exit(1)
312 312 else:
313 313 return self.subapp.start()
@@ -1,224 +1,224 b''
1 1 # encoding: utf-8
2 2 """An object for managing IPython profile directories."""
3 3
4 4 # Copyright (c) IPython Development Team.
5 5 # Distributed under the terms of the Modified BSD License.
6 6
7 7 import os
8 8 import shutil
9 9 import errno
10 10
11 11 from traitlets.config.configurable import LoggingConfigurable
12 12 from IPython.paths import get_ipython_package_dir
13 13 from IPython.utils.path import expand_path, ensure_dir_exists
14 14 from IPython.utils import py3compat
15 15 from traitlets import Unicode, Bool, observe
16 16
17 17 #-----------------------------------------------------------------------------
18 18 # Module errors
19 19 #-----------------------------------------------------------------------------
20 20
21 21 class ProfileDirError(Exception):
22 22 pass
23 23
24 24
25 25 #-----------------------------------------------------------------------------
26 26 # Class for managing profile directories
27 27 #-----------------------------------------------------------------------------
28 28
29 29 class ProfileDir(LoggingConfigurable):
30 30 """An object to manage the profile directory and its resources.
31 31
32 32 The profile directory is used by all IPython applications, to manage
33 33 configuration, logging and security.
34 34
35 35 This object knows how to find, create and manage these directories. This
36 36 should be used by any code that wants to handle profiles.
37 37 """
38 38
39 39 security_dir_name = Unicode('security')
40 40 log_dir_name = Unicode('log')
41 41 startup_dir_name = Unicode('startup')
42 42 pid_dir_name = Unicode('pid')
43 43 static_dir_name = Unicode('static')
44 44 security_dir = Unicode(u'')
45 45 log_dir = Unicode(u'')
46 46 startup_dir = Unicode(u'')
47 47 pid_dir = Unicode(u'')
48 48 static_dir = Unicode(u'')
49 49
50 50 location = Unicode(u'',
51 51 help="""Set the profile location directly. This overrides the logic used by the
52 52 `profile` option.""",
53 53 ).tag(config=True)
54 54
55 55 _location_isset = Bool(False) # flag for detecting multiply set location
56 56 @observe('location')
57 57 def _location_changed(self, change):
58 58 if self._location_isset:
59 59 raise RuntimeError("Cannot set profile location more than once.")
60 60 self._location_isset = True
61 61 new = change['new']
62 62 ensure_dir_exists(new)
63 63
64 64 # ensure config files exist:
65 65 self.security_dir = os.path.join(new, self.security_dir_name)
66 66 self.log_dir = os.path.join(new, self.log_dir_name)
67 67 self.startup_dir = os.path.join(new, self.startup_dir_name)
68 68 self.pid_dir = os.path.join(new, self.pid_dir_name)
69 69 self.static_dir = os.path.join(new, self.static_dir_name)
70 70 self.check_dirs()
71 71
72 72 def _mkdir(self, path, mode=None):
73 73 """ensure a directory exists at a given path
74 74
75 75 This is a version of os.mkdir, with the following differences:
76 76
77 77 - returns True if it created the directory, False otherwise
78 78 - ignores EEXIST, protecting against race conditions where
79 79 the dir may have been created in between the check and
80 80 the creation
81 81 - sets permissions if requested and the dir already exists
82 82 """
83 83 if os.path.exists(path):
84 84 if mode and os.stat(path).st_mode != mode:
85 85 try:
86 86 os.chmod(path, mode)
87 87 except OSError:
88 88 self.log.warning(
89 89 "Could not set permissions on %s",
90 90 path
91 91 )
92 92 return False
93 93 try:
94 94 if mode:
95 95 os.mkdir(path, mode)
96 96 else:
97 97 os.mkdir(path)
98 98 except OSError as e:
99 99 if e.errno == errno.EEXIST:
100 100 return False
101 101 else:
102 102 raise
103 103
104 104 return True
105 105
106 106 @observe('log_dir')
107 107 def check_log_dir(self, change=None):
108 108 self._mkdir(self.log_dir)
109 109
110 110 @observe('startup_dir')
111 111 def check_startup_dir(self, change=None):
112 112 self._mkdir(self.startup_dir)
113 113
114 114 readme = os.path.join(self.startup_dir, 'README')
115 115 src = os.path.join(get_ipython_package_dir(), u'core', u'profile', u'README_STARTUP')
116 116
117 117 if not os.path.exists(src):
118 118 self.log.warning("Could not copy README_STARTUP to startup dir. Source file %s does not exist.", src)
119 119
120 120 if os.path.exists(src) and not os.path.exists(readme):
121 121 shutil.copy(src, readme)
122 122
123 123 @observe('security_dir')
124 124 def check_security_dir(self, change=None):
125 125 self._mkdir(self.security_dir, 0o40700)
126 126
127 127 @observe('pid_dir')
128 128 def check_pid_dir(self, change=None):
129 129 self._mkdir(self.pid_dir, 0o40700)
130 130
131 131 def check_dirs(self):
132 132 self.check_security_dir()
133 133 self.check_log_dir()
134 134 self.check_pid_dir()
135 135 self.check_startup_dir()
136 136
137 137 def copy_config_file(self, config_file, path=None, overwrite=False):
138 138 """Copy a default config file into the active profile directory.
139 139
140 140 Default configuration files are kept in :mod:`IPython.core.profile`.
141 141 This function moves these from that location to the working profile
142 142 directory.
143 143 """
144 144 dst = os.path.join(self.location, config_file)
145 145 if os.path.isfile(dst) and not overwrite:
146 146 return False
147 147 if path is None:
148 148 path = os.path.join(get_ipython_package_dir(), u'core', u'profile', u'default')
149 149 src = os.path.join(path, config_file)
150 150 shutil.copy(src, dst)
151 151 return True
152 152
153 153 @classmethod
154 154 def create_profile_dir(cls, profile_dir, config=None):
155 155 """Create a new profile directory given a full path.
156 156
157 157 Parameters
158 158 ----------
159 159 profile_dir : str
160 160 The full path to the profile directory. If it does exist, it will
161 161 be used. If not, it will be created.
162 162 """
163 163 return cls(location=profile_dir, config=config)
164 164
165 165 @classmethod
166 166 def create_profile_dir_by_name(cls, path, name=u'default', config=None):
167 167 """Create a profile dir by profile name and path.
168 168
169 169 Parameters
170 170 ----------
171 171 path : unicode
172 172 The path (directory) to put the profile directory in.
173 173 name : unicode
174 174 The name of the profile. The name of the profile directory will
175 175 be "profile_<profile>".
176 176 """
177 177 if not os.path.isdir(path):
178 178 raise ProfileDirError('Directory not found: %s' % path)
179 179 profile_dir = os.path.join(path, u'profile_' + name)
180 180 return cls(location=profile_dir, config=config)
181 181
182 182 @classmethod
183 183 def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None):
184 184 """Find an existing profile dir by profile name, return its ProfileDir.
185 185
186 186 This searches through a sequence of paths for a profile dir. If it
187 187 is not found, a :class:`ProfileDirError` exception will be raised.
188 188
189 189 The search path algorithm is:
190 1. ``py3compat.getcwd()``
190 1. ``os.getcwd()``
191 191 2. ``ipython_dir``
192 192
193 193 Parameters
194 194 ----------
195 195 ipython_dir : unicode or str
196 196 The IPython directory to use.
197 197 name : unicode or str
198 198 The name of the profile. The name of the profile directory
199 199 will be "profile_<profile>".
200 200 """
201 201 dirname = u'profile_' + name
202 paths = [py3compat.getcwd(), ipython_dir]
202 paths = [os.getcwd(), ipython_dir]
203 203 for p in paths:
204 204 profile_dir = os.path.join(p, dirname)
205 205 if os.path.isdir(profile_dir):
206 206 return cls(location=profile_dir, config=config)
207 207 else:
208 208 raise ProfileDirError('Profile directory not found in paths: %s' % dirname)
209 209
210 210 @classmethod
211 211 def find_profile_dir(cls, profile_dir, config=None):
212 212 """Find/create a profile dir and return its ProfileDir.
213 213
214 214 This will create the profile directory if it doesn't exist.
215 215
216 216 Parameters
217 217 ----------
218 218 profile_dir : unicode or str
219 219 The path of the profile directory.
220 220 """
221 221 profile_dir = expand_path(profile_dir)
222 222 if not os.path.isdir(profile_dir):
223 223 raise ProfileDirError('Profile directory not found: %s' % profile_dir)
224 224 return cls(location=profile_dir, config=config)
@@ -1,74 +1,74 b''
1 1 # coding: utf-8
2 2 """Tests for IPython.core.application"""
3 3
4 4 import os
5 5 import tempfile
6 6
7 7 import nose.tools as nt
8 8
9 9 from traitlets import Unicode
10 10
11 11 from IPython.core.application import BaseIPythonApplication
12 12 from IPython.testing import decorators as dec
13 13 from IPython.utils import py3compat
14 14 from IPython.utils.tempdir import TemporaryDirectory
15 15
16 16
17 17 @dec.onlyif_unicode_paths
18 18 def test_unicode_cwd():
19 19 """Check that IPython starts with non-ascii characters in the path."""
20 20 wd = tempfile.mkdtemp(suffix=u"€")
21 21
22 old_wd = py3compat.getcwd()
22 old_wd = os.getcwd()
23 23 os.chdir(wd)
24 #raise Exception(repr(py3compat.getcwd()))
24 #raise Exception(repr(os.getcwd()))
25 25 try:
26 26 app = BaseIPythonApplication()
27 27 # The lines below are copied from Application.initialize()
28 28 app.init_profile_dir()
29 29 app.init_config_files()
30 30 app.load_config_file(suppress_errors=False)
31 31 finally:
32 32 os.chdir(old_wd)
33 33
34 34 @dec.onlyif_unicode_paths
35 35 def test_unicode_ipdir():
36 36 """Check that IPython starts with non-ascii characters in the IP dir."""
37 37 ipdir = tempfile.mkdtemp(suffix=u"€")
38 38
39 39 # Create the config file, so it tries to load it.
40 40 with open(os.path.join(ipdir, 'ipython_config.py'), "w") as f:
41 41 pass
42 42
43 43 old_ipdir1 = os.environ.pop("IPYTHONDIR", None)
44 44 old_ipdir2 = os.environ.pop("IPYTHON_DIR", None)
45 45 os.environ["IPYTHONDIR"] = ipdir
46 46 try:
47 47 app = BaseIPythonApplication()
48 48 # The lines below are copied from Application.initialize()
49 49 app.init_profile_dir()
50 50 app.init_config_files()
51 51 app.load_config_file(suppress_errors=False)
52 52 finally:
53 53 if old_ipdir1:
54 54 os.environ["IPYTHONDIR"] = old_ipdir1
55 55 if old_ipdir2:
56 56 os.environ["IPYTHONDIR"] = old_ipdir2
57 57
58 58 def test_cli_priority():
59 59 with TemporaryDirectory() as td:
60 60
61 61 class TestApp(BaseIPythonApplication):
62 62 test = Unicode().tag(config=True)
63 63
64 64 # Create the config file, so it tries to load it.
65 65 with open(os.path.join(td, 'ipython_config.py'), "w") as f:
66 66 f.write("c.TestApp.test = 'config file'")
67 67
68 68 app = TestApp()
69 69 app.initialize(['--profile-dir', td])
70 70 nt.assert_equal(app.test, 'config file')
71 71 app = TestApp()
72 72 app.initialize(['--profile-dir', td, '--TestApp.test=cli'])
73 73 nt.assert_equal(app.test, 'cli')
74 74
@@ -1,162 +1,162 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for completerlib.
3 3
4 4 """
5 5
6 6 #-----------------------------------------------------------------------------
7 7 # Imports
8 8 #-----------------------------------------------------------------------------
9 9
10 10 import os
11 11 import shutil
12 12 import sys
13 13 import tempfile
14 14 import unittest
15 15 from os.path import join
16 16
17 17 import nose.tools as nt
18 18
19 19 from IPython.core.completerlib import magic_run_completer, module_completion
20 20 from IPython.utils import py3compat
21 21 from IPython.utils.tempdir import TemporaryDirectory
22 22 from IPython.testing.decorators import onlyif_unicode_paths
23 23
24 24
25 25 class MockEvent(object):
26 26 def __init__(self, line):
27 27 self.line = line
28 28
29 29 #-----------------------------------------------------------------------------
30 30 # Test functions begin
31 31 #-----------------------------------------------------------------------------
32 32 class Test_magic_run_completer(unittest.TestCase):
33 33 files = [u"aao.py", u"a.py", u"b.py", u"aao.txt"]
34 34 dirs = [u"adir/", "bdir/"]
35 35
36 36 def setUp(self):
37 37 self.BASETESTDIR = tempfile.mkdtemp()
38 38 for fil in self.files:
39 39 with open(join(self.BASETESTDIR, fil), "w") as sfile:
40 40 sfile.write("pass\n")
41 41 for d in self.dirs:
42 42 os.mkdir(join(self.BASETESTDIR, d))
43 43
44 self.oldpath = py3compat.getcwd()
44 self.oldpath = os.getcwd()
45 45 os.chdir(self.BASETESTDIR)
46 46
47 47 def tearDown(self):
48 48 os.chdir(self.oldpath)
49 49 shutil.rmtree(self.BASETESTDIR)
50 50
51 51 def test_1(self):
52 52 """Test magic_run_completer, should match two alterntives
53 53 """
54 54 event = MockEvent(u"%run a")
55 55 mockself = None
56 56 match = set(magic_run_completer(mockself, event))
57 57 self.assertEqual(match, {u"a.py", u"aao.py", u"adir/"})
58 58
59 59 def test_2(self):
60 60 """Test magic_run_completer, should match one alterntive
61 61 """
62 62 event = MockEvent(u"%run aa")
63 63 mockself = None
64 64 match = set(magic_run_completer(mockself, event))
65 65 self.assertEqual(match, {u"aao.py"})
66 66
67 67 def test_3(self):
68 68 """Test magic_run_completer with unterminated " """
69 69 event = MockEvent(u'%run "a')
70 70 mockself = None
71 71 match = set(magic_run_completer(mockself, event))
72 72 self.assertEqual(match, {u"a.py", u"aao.py", u"adir/"})
73 73
74 74 def test_completion_more_args(self):
75 75 event = MockEvent(u'%run a.py ')
76 76 match = set(magic_run_completer(None, event))
77 77 self.assertEqual(match, set(self.files + self.dirs))
78 78
79 79 def test_completion_in_dir(self):
80 80 # Github issue #3459
81 81 event = MockEvent(u'%run a.py {}'.format(join(self.BASETESTDIR, 'a')))
82 82 print(repr(event.line))
83 83 match = set(magic_run_completer(None, event))
84 84 # We specifically use replace here rather than normpath, because
85 85 # at one point there were duplicates 'adir' and 'adir/', and normpath
86 86 # would hide the failure for that.
87 87 self.assertEqual(match, {join(self.BASETESTDIR, f).replace('\\','/')
88 88 for f in (u'a.py', u'aao.py', u'aao.txt', u'adir/')})
89 89
90 90 class Test_magic_run_completer_nonascii(unittest.TestCase):
91 91 @onlyif_unicode_paths
92 92 def setUp(self):
93 93 self.BASETESTDIR = tempfile.mkdtemp()
94 94 for fil in [u"aaΓΈ.py", u"a.py", u"b.py"]:
95 95 with open(join(self.BASETESTDIR, fil), "w") as sfile:
96 96 sfile.write("pass\n")
97 self.oldpath = py3compat.getcwd()
97 self.oldpath = os.getcwd()
98 98 os.chdir(self.BASETESTDIR)
99 99
100 100 def tearDown(self):
101 101 os.chdir(self.oldpath)
102 102 shutil.rmtree(self.BASETESTDIR)
103 103
104 104 @onlyif_unicode_paths
105 105 def test_1(self):
106 106 """Test magic_run_completer, should match two alterntives
107 107 """
108 108 event = MockEvent(u"%run a")
109 109 mockself = None
110 110 match = set(magic_run_completer(mockself, event))
111 111 self.assertEqual(match, {u"a.py", u"aaΓΈ.py"})
112 112
113 113 @onlyif_unicode_paths
114 114 def test_2(self):
115 115 """Test magic_run_completer, should match one alterntive
116 116 """
117 117 event = MockEvent(u"%run aa")
118 118 mockself = None
119 119 match = set(magic_run_completer(mockself, event))
120 120 self.assertEqual(match, {u"aaΓΈ.py"})
121 121
122 122 @onlyif_unicode_paths
123 123 def test_3(self):
124 124 """Test magic_run_completer with unterminated " """
125 125 event = MockEvent(u'%run "a')
126 126 mockself = None
127 127 match = set(magic_run_completer(mockself, event))
128 128 self.assertEqual(match, {u"a.py", u"aaΓΈ.py"})
129 129
130 130 # module_completer:
131 131
132 132 def test_import_invalid_module():
133 133 """Testing of issue https://github.com/ipython/ipython/issues/1107"""
134 134 invalid_module_names = {'foo-bar', 'foo:bar', '10foo'}
135 135 valid_module_names = {'foobar'}
136 136 with TemporaryDirectory() as tmpdir:
137 137 sys.path.insert( 0, tmpdir )
138 138 for name in invalid_module_names | valid_module_names:
139 139 filename = os.path.join(tmpdir, name + '.py')
140 140 open(filename, 'w').close()
141 141
142 142 s = set( module_completion('import foo') )
143 143 intersection = s.intersection(invalid_module_names)
144 144 nt.assert_equal(intersection, set())
145 145
146 146 assert valid_module_names.issubset(s), valid_module_names.intersection(s)
147 147
148 148
149 149 def test_bad_module_all():
150 150 """Test module with invalid __all__
151 151
152 152 https://github.com/ipython/ipython/issues/9678
153 153 """
154 154 testsdir = os.path.dirname(__file__)
155 155 sys.path.insert(0, testsdir)
156 156 try:
157 157 results = module_completion('from bad_all import ')
158 158 nt.assert_in('puppies', results)
159 159 for r in results:
160 160 nt.assert_is_instance(r, str)
161 161 finally:
162 162 sys.path.remove(testsdir)
@@ -1,906 +1,906 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for the key interactiveshell module.
3 3
4 4 Historically the main classes in interactiveshell have been under-tested. This
5 5 module should grow as many single-method tests as possible to trap many of the
6 6 recurring bugs we seem to encounter with high-level interaction.
7 7 """
8 8
9 9 # Copyright (c) IPython Development Team.
10 10 # Distributed under the terms of the Modified BSD License.
11 11
12 12 import ast
13 13 import os
14 14 import signal
15 15 import shutil
16 16 import sys
17 17 import tempfile
18 18 import unittest
19 19 try:
20 20 from unittest import mock
21 21 except ImportError:
22 22 import mock
23 23 from os.path import join
24 24
25 25 import nose.tools as nt
26 26
27 27 from IPython.core.error import InputRejected
28 28 from IPython.core.inputtransformer import InputTransformer
29 29 from IPython.testing.decorators import (
30 30 skipif, skip_win32, onlyif_unicode_paths, onlyif_cmds_exist,
31 31 )
32 32 from IPython.testing import tools as tt
33 33 from IPython.utils.process import find_cmd
34 34 from IPython.utils import py3compat
35 35 from IPython.utils.py3compat import PY3
36 36
37 37 if PY3:
38 38 from io import StringIO
39 39 else:
40 40 from StringIO import StringIO
41 41
42 42 #-----------------------------------------------------------------------------
43 43 # Globals
44 44 #-----------------------------------------------------------------------------
45 45 # This is used by every single test, no point repeating it ad nauseam
46 46 ip = get_ipython()
47 47
48 48 #-----------------------------------------------------------------------------
49 49 # Tests
50 50 #-----------------------------------------------------------------------------
51 51
52 52 class DerivedInterrupt(KeyboardInterrupt):
53 53 pass
54 54
55 55 class InteractiveShellTestCase(unittest.TestCase):
56 56 def test_naked_string_cells(self):
57 57 """Test that cells with only naked strings are fully executed"""
58 58 # First, single-line inputs
59 59 ip.run_cell('"a"\n')
60 60 self.assertEqual(ip.user_ns['_'], 'a')
61 61 # And also multi-line cells
62 62 ip.run_cell('"""a\nb"""\n')
63 63 self.assertEqual(ip.user_ns['_'], 'a\nb')
64 64
65 65 def test_run_empty_cell(self):
66 66 """Just make sure we don't get a horrible error with a blank
67 67 cell of input. Yes, I did overlook that."""
68 68 old_xc = ip.execution_count
69 69 res = ip.run_cell('')
70 70 self.assertEqual(ip.execution_count, old_xc)
71 71 self.assertEqual(res.execution_count, None)
72 72
73 73 def test_run_cell_multiline(self):
74 74 """Multi-block, multi-line cells must execute correctly.
75 75 """
76 76 src = '\n'.join(["x=1",
77 77 "y=2",
78 78 "if 1:",
79 79 " x += 1",
80 80 " y += 1",])
81 81 res = ip.run_cell(src)
82 82 self.assertEqual(ip.user_ns['x'], 2)
83 83 self.assertEqual(ip.user_ns['y'], 3)
84 84 self.assertEqual(res.success, True)
85 85 self.assertEqual(res.result, None)
86 86
87 87 def test_multiline_string_cells(self):
88 88 "Code sprinkled with multiline strings should execute (GH-306)"
89 89 ip.run_cell('tmp=0')
90 90 self.assertEqual(ip.user_ns['tmp'], 0)
91 91 res = ip.run_cell('tmp=1;"""a\nb"""\n')
92 92 self.assertEqual(ip.user_ns['tmp'], 1)
93 93 self.assertEqual(res.success, True)
94 94 self.assertEqual(res.result, "a\nb")
95 95
96 96 def test_dont_cache_with_semicolon(self):
97 97 "Ending a line with semicolon should not cache the returned object (GH-307)"
98 98 oldlen = len(ip.user_ns['Out'])
99 99 for cell in ['1;', '1;1;']:
100 100 res = ip.run_cell(cell, store_history=True)
101 101 newlen = len(ip.user_ns['Out'])
102 102 self.assertEqual(oldlen, newlen)
103 103 self.assertIsNone(res.result)
104 104 i = 0
105 105 #also test the default caching behavior
106 106 for cell in ['1', '1;1']:
107 107 ip.run_cell(cell, store_history=True)
108 108 newlen = len(ip.user_ns['Out'])
109 109 i += 1
110 110 self.assertEqual(oldlen+i, newlen)
111 111
112 112 def test_syntax_error(self):
113 113 res = ip.run_cell("raise = 3")
114 114 self.assertIsInstance(res.error_before_exec, SyntaxError)
115 115
116 116 def test_In_variable(self):
117 117 "Verify that In variable grows with user input (GH-284)"
118 118 oldlen = len(ip.user_ns['In'])
119 119 ip.run_cell('1;', store_history=True)
120 120 newlen = len(ip.user_ns['In'])
121 121 self.assertEqual(oldlen+1, newlen)
122 122 self.assertEqual(ip.user_ns['In'][-1],'1;')
123 123
124 124 def test_magic_names_in_string(self):
125 125 ip.run_cell('a = """\n%exit\n"""')
126 126 self.assertEqual(ip.user_ns['a'], '\n%exit\n')
127 127
128 128 def test_trailing_newline(self):
129 129 """test that running !(command) does not raise a SyntaxError"""
130 130 ip.run_cell('!(true)\n', False)
131 131 ip.run_cell('!(true)\n\n\n', False)
132 132
133 133 def test_gh_597(self):
134 134 """Pretty-printing lists of objects with non-ascii reprs may cause
135 135 problems."""
136 136 class Spam(object):
137 137 def __repr__(self):
138 138 return "\xe9"*50
139 139 import IPython.core.formatters
140 140 f = IPython.core.formatters.PlainTextFormatter()
141 141 f([Spam(),Spam()])
142 142
143 143
144 144 def test_future_flags(self):
145 145 """Check that future flags are used for parsing code (gh-777)"""
146 146 ip.run_cell('from __future__ import barry_as_FLUFL')
147 147 try:
148 148 ip.run_cell('prfunc_return_val = 1 <> 2')
149 149 assert 'prfunc_return_val' in ip.user_ns
150 150 finally:
151 151 # Reset compiler flags so we don't mess up other tests.
152 152 ip.compile.reset_compiler_flags()
153 153
154 154 def test_can_pickle(self):
155 155 "Can we pickle objects defined interactively (GH-29)"
156 156 ip = get_ipython()
157 157 ip.reset()
158 158 ip.run_cell(("class Mylist(list):\n"
159 159 " def __init__(self,x=[]):\n"
160 160 " list.__init__(self,x)"))
161 161 ip.run_cell("w=Mylist([1,2,3])")
162 162
163 163 from pickle import dumps
164 164
165 165 # We need to swap in our main module - this is only necessary
166 166 # inside the test framework, because IPython puts the interactive module
167 167 # in place (but the test framework undoes this).
168 168 _main = sys.modules['__main__']
169 169 sys.modules['__main__'] = ip.user_module
170 170 try:
171 171 res = dumps(ip.user_ns["w"])
172 172 finally:
173 173 sys.modules['__main__'] = _main
174 174 self.assertTrue(isinstance(res, bytes))
175 175
176 176 def test_global_ns(self):
177 177 "Code in functions must be able to access variables outside them."
178 178 ip = get_ipython()
179 179 ip.run_cell("a = 10")
180 180 ip.run_cell(("def f(x):\n"
181 181 " return x + a"))
182 182 ip.run_cell("b = f(12)")
183 183 self.assertEqual(ip.user_ns["b"], 22)
184 184
185 185 def test_bad_custom_tb(self):
186 186 """Check that InteractiveShell is protected from bad custom exception handlers"""
187 187 ip.set_custom_exc((IOError,), lambda etype,value,tb: 1/0)
188 188 self.assertEqual(ip.custom_exceptions, (IOError,))
189 189 with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
190 190 ip.run_cell(u'raise IOError("foo")')
191 191 self.assertEqual(ip.custom_exceptions, ())
192 192
193 193 def test_bad_custom_tb_return(self):
194 194 """Check that InteractiveShell is protected from bad return types in custom exception handlers"""
195 195 ip.set_custom_exc((NameError,),lambda etype,value,tb, tb_offset=None: 1)
196 196 self.assertEqual(ip.custom_exceptions, (NameError,))
197 197 with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
198 198 ip.run_cell(u'a=abracadabra')
199 199 self.assertEqual(ip.custom_exceptions, ())
200 200
201 201 def test_drop_by_id(self):
202 202 myvars = {"a":object(), "b":object(), "c": object()}
203 203 ip.push(myvars, interactive=False)
204 204 for name in myvars:
205 205 assert name in ip.user_ns, name
206 206 assert name in ip.user_ns_hidden, name
207 207 ip.user_ns['b'] = 12
208 208 ip.drop_by_id(myvars)
209 209 for name in ["a", "c"]:
210 210 assert name not in ip.user_ns, name
211 211 assert name not in ip.user_ns_hidden, name
212 212 assert ip.user_ns['b'] == 12
213 213 ip.reset()
214 214
215 215 def test_var_expand(self):
216 216 ip.user_ns['f'] = u'Ca\xf1o'
217 217 self.assertEqual(ip.var_expand(u'echo $f'), u'echo Ca\xf1o')
218 218 self.assertEqual(ip.var_expand(u'echo {f}'), u'echo Ca\xf1o')
219 219 self.assertEqual(ip.var_expand(u'echo {f[:-1]}'), u'echo Ca\xf1')
220 220 self.assertEqual(ip.var_expand(u'echo {1*2}'), u'echo 2')
221 221
222 222 ip.user_ns['f'] = b'Ca\xc3\xb1o'
223 223 # This should not raise any exception:
224 224 ip.var_expand(u'echo $f')
225 225
226 226 def test_var_expand_local(self):
227 227 """Test local variable expansion in !system and %magic calls"""
228 228 # !system
229 229 ip.run_cell('def test():\n'
230 230 ' lvar = "ttt"\n'
231 231 ' ret = !echo {lvar}\n'
232 232 ' return ret[0]\n')
233 233 res = ip.user_ns['test']()
234 234 nt.assert_in('ttt', res)
235 235
236 236 # %magic
237 237 ip.run_cell('def makemacro():\n'
238 238 ' macroname = "macro_var_expand_locals"\n'
239 239 ' %macro {macroname} codestr\n')
240 240 ip.user_ns['codestr'] = "str(12)"
241 241 ip.run_cell('makemacro()')
242 242 nt.assert_in('macro_var_expand_locals', ip.user_ns)
243 243
244 244 def test_var_expand_self(self):
245 245 """Test variable expansion with the name 'self', which was failing.
246 246
247 247 See https://github.com/ipython/ipython/issues/1878#issuecomment-7698218
248 248 """
249 249 ip.run_cell('class cTest:\n'
250 250 ' classvar="see me"\n'
251 251 ' def test(self):\n'
252 252 ' res = !echo Variable: {self.classvar}\n'
253 253 ' return res[0]\n')
254 254 nt.assert_in('see me', ip.user_ns['cTest']().test())
255 255
256 256 def test_bad_var_expand(self):
257 257 """var_expand on invalid formats shouldn't raise"""
258 258 # SyntaxError
259 259 self.assertEqual(ip.var_expand(u"{'a':5}"), u"{'a':5}")
260 260 # NameError
261 261 self.assertEqual(ip.var_expand(u"{asdf}"), u"{asdf}")
262 262 # ZeroDivisionError
263 263 self.assertEqual(ip.var_expand(u"{1/0}"), u"{1/0}")
264 264
265 265 def test_silent_postexec(self):
266 266 """run_cell(silent=True) doesn't invoke pre/post_run_cell callbacks"""
267 267 pre_explicit = mock.Mock()
268 268 pre_always = mock.Mock()
269 269 post_explicit = mock.Mock()
270 270 post_always = mock.Mock()
271 271
272 272 ip.events.register('pre_run_cell', pre_explicit)
273 273 ip.events.register('pre_execute', pre_always)
274 274 ip.events.register('post_run_cell', post_explicit)
275 275 ip.events.register('post_execute', post_always)
276 276
277 277 try:
278 278 ip.run_cell("1", silent=True)
279 279 assert pre_always.called
280 280 assert not pre_explicit.called
281 281 assert post_always.called
282 282 assert not post_explicit.called
283 283 # double-check that non-silent exec did what we expected
284 284 # silent to avoid
285 285 ip.run_cell("1")
286 286 assert pre_explicit.called
287 287 assert post_explicit.called
288 288 finally:
289 289 # remove post-exec
290 290 ip.events.unregister('pre_run_cell', pre_explicit)
291 291 ip.events.unregister('pre_execute', pre_always)
292 292 ip.events.unregister('post_run_cell', post_explicit)
293 293 ip.events.unregister('post_execute', post_always)
294 294
295 295 def test_silent_noadvance(self):
296 296 """run_cell(silent=True) doesn't advance execution_count"""
297 297 ec = ip.execution_count
298 298 # silent should force store_history=False
299 299 ip.run_cell("1", store_history=True, silent=True)
300 300
301 301 self.assertEqual(ec, ip.execution_count)
302 302 # double-check that non-silent exec did what we expected
303 303 # silent to avoid
304 304 ip.run_cell("1", store_history=True)
305 305 self.assertEqual(ec+1, ip.execution_count)
306 306
307 307 def test_silent_nodisplayhook(self):
308 308 """run_cell(silent=True) doesn't trigger displayhook"""
309 309 d = dict(called=False)
310 310
311 311 trap = ip.display_trap
312 312 save_hook = trap.hook
313 313
314 314 def failing_hook(*args, **kwargs):
315 315 d['called'] = True
316 316
317 317 try:
318 318 trap.hook = failing_hook
319 319 res = ip.run_cell("1", silent=True)
320 320 self.assertFalse(d['called'])
321 321 self.assertIsNone(res.result)
322 322 # double-check that non-silent exec did what we expected
323 323 # silent to avoid
324 324 ip.run_cell("1")
325 325 self.assertTrue(d['called'])
326 326 finally:
327 327 trap.hook = save_hook
328 328
329 329 def test_ofind_line_magic(self):
330 330 from IPython.core.magic import register_line_magic
331 331
332 332 @register_line_magic
333 333 def lmagic(line):
334 334 "A line magic"
335 335
336 336 # Get info on line magic
337 337 lfind = ip._ofind('lmagic')
338 338 info = dict(found=True, isalias=False, ismagic=True,
339 339 namespace = 'IPython internal', obj= lmagic.__wrapped__,
340 340 parent = None)
341 341 nt.assert_equal(lfind, info)
342 342
343 343 def test_ofind_cell_magic(self):
344 344 from IPython.core.magic import register_cell_magic
345 345
346 346 @register_cell_magic
347 347 def cmagic(line, cell):
348 348 "A cell magic"
349 349
350 350 # Get info on cell magic
351 351 find = ip._ofind('cmagic')
352 352 info = dict(found=True, isalias=False, ismagic=True,
353 353 namespace = 'IPython internal', obj= cmagic.__wrapped__,
354 354 parent = None)
355 355 nt.assert_equal(find, info)
356 356
357 357 def test_ofind_property_with_error(self):
358 358 class A(object):
359 359 @property
360 360 def foo(self):
361 361 raise NotImplementedError()
362 362 a = A()
363 363
364 364 found = ip._ofind('a.foo', [('locals', locals())])
365 365 info = dict(found=True, isalias=False, ismagic=False,
366 366 namespace='locals', obj=A.foo, parent=a)
367 367 nt.assert_equal(found, info)
368 368
369 369 def test_ofind_multiple_attribute_lookups(self):
370 370 class A(object):
371 371 @property
372 372 def foo(self):
373 373 raise NotImplementedError()
374 374
375 375 a = A()
376 376 a.a = A()
377 377 a.a.a = A()
378 378
379 379 found = ip._ofind('a.a.a.foo', [('locals', locals())])
380 380 info = dict(found=True, isalias=False, ismagic=False,
381 381 namespace='locals', obj=A.foo, parent=a.a.a)
382 382 nt.assert_equal(found, info)
383 383
384 384 def test_ofind_slotted_attributes(self):
385 385 class A(object):
386 386 __slots__ = ['foo']
387 387 def __init__(self):
388 388 self.foo = 'bar'
389 389
390 390 a = A()
391 391 found = ip._ofind('a.foo', [('locals', locals())])
392 392 info = dict(found=True, isalias=False, ismagic=False,
393 393 namespace='locals', obj=a.foo, parent=a)
394 394 nt.assert_equal(found, info)
395 395
396 396 found = ip._ofind('a.bar', [('locals', locals())])
397 397 info = dict(found=False, isalias=False, ismagic=False,
398 398 namespace=None, obj=None, parent=a)
399 399 nt.assert_equal(found, info)
400 400
401 401 def test_ofind_prefers_property_to_instance_level_attribute(self):
402 402 class A(object):
403 403 @property
404 404 def foo(self):
405 405 return 'bar'
406 406 a = A()
407 407 a.__dict__['foo'] = 'baz'
408 408 nt.assert_equal(a.foo, 'bar')
409 409 found = ip._ofind('a.foo', [('locals', locals())])
410 410 nt.assert_is(found['obj'], A.foo)
411 411
412 412 def test_custom_syntaxerror_exception(self):
413 413 called = []
414 414 def my_handler(shell, etype, value, tb, tb_offset=None):
415 415 called.append(etype)
416 416 shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
417 417
418 418 ip.set_custom_exc((SyntaxError,), my_handler)
419 419 try:
420 420 ip.run_cell("1f")
421 421 # Check that this was called, and only once.
422 422 self.assertEqual(called, [SyntaxError])
423 423 finally:
424 424 # Reset the custom exception hook
425 425 ip.set_custom_exc((), None)
426 426
427 427 def test_custom_exception(self):
428 428 called = []
429 429 def my_handler(shell, etype, value, tb, tb_offset=None):
430 430 called.append(etype)
431 431 shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
432 432
433 433 ip.set_custom_exc((ValueError,), my_handler)
434 434 try:
435 435 res = ip.run_cell("raise ValueError('test')")
436 436 # Check that this was called, and only once.
437 437 self.assertEqual(called, [ValueError])
438 438 # Check that the error is on the result object
439 439 self.assertIsInstance(res.error_in_exec, ValueError)
440 440 finally:
441 441 # Reset the custom exception hook
442 442 ip.set_custom_exc((), None)
443 443
444 444 def test_mktempfile(self):
445 445 filename = ip.mktempfile()
446 446 # Check that we can open the file again on Windows
447 447 with open(filename, 'w') as f:
448 448 f.write('abc')
449 449
450 450 filename = ip.mktempfile(data='blah')
451 451 with open(filename, 'r') as f:
452 452 self.assertEqual(f.read(), 'blah')
453 453
454 454 def test_new_main_mod(self):
455 455 # Smoketest to check that this accepts a unicode module name
456 456 name = u'jiefmw'
457 457 mod = ip.new_main_mod(u'%s.py' % name, name)
458 458 self.assertEqual(mod.__name__, name)
459 459
460 460 def test_get_exception_only(self):
461 461 try:
462 462 raise KeyboardInterrupt
463 463 except KeyboardInterrupt:
464 464 msg = ip.get_exception_only()
465 465 self.assertEqual(msg, 'KeyboardInterrupt\n')
466 466
467 467 try:
468 468 raise DerivedInterrupt("foo")
469 469 except KeyboardInterrupt:
470 470 msg = ip.get_exception_only()
471 471 self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n')
472 472
473 473 def test_inspect_text(self):
474 474 ip.run_cell('a = 5')
475 475 text = ip.object_inspect_text('a')
476 476 self.assertIsInstance(text, str)
477 477
478 478
479 479 class TestSafeExecfileNonAsciiPath(unittest.TestCase):
480 480
481 481 @onlyif_unicode_paths
482 482 def setUp(self):
483 483 self.BASETESTDIR = tempfile.mkdtemp()
484 484 self.TESTDIR = join(self.BASETESTDIR, u"Γ₯Àâ")
485 485 os.mkdir(self.TESTDIR)
486 486 with open(join(self.TESTDIR, u"Γ₯Àâtestscript.py"), "w") as sfile:
487 487 sfile.write("pass\n")
488 self.oldpath = py3compat.getcwd()
488 self.oldpath = os.getcwd()
489 489 os.chdir(self.TESTDIR)
490 490 self.fname = u"Γ₯Àâtestscript.py"
491 491
492 492 def tearDown(self):
493 493 os.chdir(self.oldpath)
494 494 shutil.rmtree(self.BASETESTDIR)
495 495
496 496 @onlyif_unicode_paths
497 497 def test_1(self):
498 498 """Test safe_execfile with non-ascii path
499 499 """
500 500 ip.safe_execfile(self.fname, {}, raise_exceptions=True)
501 501
502 502 class ExitCodeChecks(tt.TempFileMixin):
503 503 def test_exit_code_ok(self):
504 504 self.system('exit 0')
505 505 self.assertEqual(ip.user_ns['_exit_code'], 0)
506 506
507 507 def test_exit_code_error(self):
508 508 self.system('exit 1')
509 509 self.assertEqual(ip.user_ns['_exit_code'], 1)
510 510
511 511 @skipif(not hasattr(signal, 'SIGALRM'))
512 512 def test_exit_code_signal(self):
513 513 self.mktmp("import signal, time\n"
514 514 "signal.setitimer(signal.ITIMER_REAL, 0.1)\n"
515 515 "time.sleep(1)\n")
516 516 self.system("%s %s" % (sys.executable, self.fname))
517 517 self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGALRM)
518 518
519 519 @onlyif_cmds_exist("csh")
520 520 def test_exit_code_signal_csh(self):
521 521 SHELL = os.environ.get('SHELL', None)
522 522 os.environ['SHELL'] = find_cmd("csh")
523 523 try:
524 524 self.test_exit_code_signal()
525 525 finally:
526 526 if SHELL is not None:
527 527 os.environ['SHELL'] = SHELL
528 528 else:
529 529 del os.environ['SHELL']
530 530
531 531 class TestSystemRaw(unittest.TestCase, ExitCodeChecks):
532 532 system = ip.system_raw
533 533
534 534 @onlyif_unicode_paths
535 535 def test_1(self):
536 536 """Test system_raw with non-ascii cmd
537 537 """
538 538 cmd = u'''python -c "'Γ₯Àâ'" '''
539 539 ip.system_raw(cmd)
540 540
541 541 @mock.patch('subprocess.call', side_effect=KeyboardInterrupt)
542 542 @mock.patch('os.system', side_effect=KeyboardInterrupt)
543 543 def test_control_c(self, *mocks):
544 544 try:
545 545 self.system("sleep 1 # wont happen")
546 546 except KeyboardInterrupt:
547 547 self.fail("system call should intercept "
548 548 "keyboard interrupt from subprocess.call")
549 549 self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGINT)
550 550
551 551 # TODO: Exit codes are currently ignored on Windows.
552 552 class TestSystemPipedExitCode(unittest.TestCase, ExitCodeChecks):
553 553 system = ip.system_piped
554 554
555 555 @skip_win32
556 556 def test_exit_code_ok(self):
557 557 ExitCodeChecks.test_exit_code_ok(self)
558 558
559 559 @skip_win32
560 560 def test_exit_code_error(self):
561 561 ExitCodeChecks.test_exit_code_error(self)
562 562
563 563 @skip_win32
564 564 def test_exit_code_signal(self):
565 565 ExitCodeChecks.test_exit_code_signal(self)
566 566
567 567 class TestModules(unittest.TestCase, tt.TempFileMixin):
568 568 def test_extraneous_loads(self):
569 569 """Test we're not loading modules on startup that we shouldn't.
570 570 """
571 571 self.mktmp("import sys\n"
572 572 "print('numpy' in sys.modules)\n"
573 573 "print('ipyparallel' in sys.modules)\n"
574 574 "print('ipykernel' in sys.modules)\n"
575 575 )
576 576 out = "False\nFalse\nFalse\n"
577 577 tt.ipexec_validate(self.fname, out)
578 578
579 579 class Negator(ast.NodeTransformer):
580 580 """Negates all number literals in an AST."""
581 581 def visit_Num(self, node):
582 582 node.n = -node.n
583 583 return node
584 584
585 585 class TestAstTransform(unittest.TestCase):
586 586 def setUp(self):
587 587 self.negator = Negator()
588 588 ip.ast_transformers.append(self.negator)
589 589
590 590 def tearDown(self):
591 591 ip.ast_transformers.remove(self.negator)
592 592
593 593 def test_run_cell(self):
594 594 with tt.AssertPrints('-34'):
595 595 ip.run_cell('print (12 + 22)')
596 596
597 597 # A named reference to a number shouldn't be transformed.
598 598 ip.user_ns['n'] = 55
599 599 with tt.AssertNotPrints('-55'):
600 600 ip.run_cell('print (n)')
601 601
602 602 def test_timeit(self):
603 603 called = set()
604 604 def f(x):
605 605 called.add(x)
606 606 ip.push({'f':f})
607 607
608 608 with tt.AssertPrints("average of "):
609 609 ip.run_line_magic("timeit", "-n1 f(1)")
610 610 self.assertEqual(called, {-1})
611 611 called.clear()
612 612
613 613 with tt.AssertPrints("average of "):
614 614 ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
615 615 self.assertEqual(called, {-2, -3})
616 616
617 617 def test_time(self):
618 618 called = []
619 619 def f(x):
620 620 called.append(x)
621 621 ip.push({'f':f})
622 622
623 623 # Test with an expression
624 624 with tt.AssertPrints("Wall time: "):
625 625 ip.run_line_magic("time", "f(5+9)")
626 626 self.assertEqual(called, [-14])
627 627 called[:] = []
628 628
629 629 # Test with a statement (different code path)
630 630 with tt.AssertPrints("Wall time: "):
631 631 ip.run_line_magic("time", "a = f(-3 + -2)")
632 632 self.assertEqual(called, [5])
633 633
634 634 def test_macro(self):
635 635 ip.push({'a':10})
636 636 # The AST transformation makes this do a+=-1
637 637 ip.define_macro("amacro", "a+=1\nprint(a)")
638 638
639 639 with tt.AssertPrints("9"):
640 640 ip.run_cell("amacro")
641 641 with tt.AssertPrints("8"):
642 642 ip.run_cell("amacro")
643 643
644 644 class IntegerWrapper(ast.NodeTransformer):
645 645 """Wraps all integers in a call to Integer()"""
646 646 def visit_Num(self, node):
647 647 if isinstance(node.n, int):
648 648 return ast.Call(func=ast.Name(id='Integer', ctx=ast.Load()),
649 649 args=[node], keywords=[])
650 650 return node
651 651
652 652 class TestAstTransform2(unittest.TestCase):
653 653 def setUp(self):
654 654 self.intwrapper = IntegerWrapper()
655 655 ip.ast_transformers.append(self.intwrapper)
656 656
657 657 self.calls = []
658 658 def Integer(*args):
659 659 self.calls.append(args)
660 660 return args
661 661 ip.push({"Integer": Integer})
662 662
663 663 def tearDown(self):
664 664 ip.ast_transformers.remove(self.intwrapper)
665 665 del ip.user_ns['Integer']
666 666
667 667 def test_run_cell(self):
668 668 ip.run_cell("n = 2")
669 669 self.assertEqual(self.calls, [(2,)])
670 670
671 671 # This shouldn't throw an error
672 672 ip.run_cell("o = 2.0")
673 673 self.assertEqual(ip.user_ns['o'], 2.0)
674 674
675 675 def test_timeit(self):
676 676 called = set()
677 677 def f(x):
678 678 called.add(x)
679 679 ip.push({'f':f})
680 680
681 681 with tt.AssertPrints("average of "):
682 682 ip.run_line_magic("timeit", "-n1 f(1)")
683 683 self.assertEqual(called, {(1,)})
684 684 called.clear()
685 685
686 686 with tt.AssertPrints("average of "):
687 687 ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
688 688 self.assertEqual(called, {(2,), (3,)})
689 689
690 690 class ErrorTransformer(ast.NodeTransformer):
691 691 """Throws an error when it sees a number."""
692 692 def visit_Num(self, node):
693 693 raise ValueError("test")
694 694
695 695 class TestAstTransformError(unittest.TestCase):
696 696 def test_unregistering(self):
697 697 err_transformer = ErrorTransformer()
698 698 ip.ast_transformers.append(err_transformer)
699 699
700 700 with tt.AssertPrints("unregister", channel='stderr'):
701 701 ip.run_cell("1 + 2")
702 702
703 703 # This should have been removed.
704 704 nt.assert_not_in(err_transformer, ip.ast_transformers)
705 705
706 706
707 707 class StringRejector(ast.NodeTransformer):
708 708 """Throws an InputRejected when it sees a string literal.
709 709
710 710 Used to verify that NodeTransformers can signal that a piece of code should
711 711 not be executed by throwing an InputRejected.
712 712 """
713 713
714 714 def visit_Str(self, node):
715 715 raise InputRejected("test")
716 716
717 717
718 718 class TestAstTransformInputRejection(unittest.TestCase):
719 719
720 720 def setUp(self):
721 721 self.transformer = StringRejector()
722 722 ip.ast_transformers.append(self.transformer)
723 723
724 724 def tearDown(self):
725 725 ip.ast_transformers.remove(self.transformer)
726 726
727 727 def test_input_rejection(self):
728 728 """Check that NodeTransformers can reject input."""
729 729
730 730 expect_exception_tb = tt.AssertPrints("InputRejected: test")
731 731 expect_no_cell_output = tt.AssertNotPrints("'unsafe'", suppress=False)
732 732
733 733 # Run the same check twice to verify that the transformer is not
734 734 # disabled after raising.
735 735 with expect_exception_tb, expect_no_cell_output:
736 736 ip.run_cell("'unsafe'")
737 737
738 738 with expect_exception_tb, expect_no_cell_output:
739 739 res = ip.run_cell("'unsafe'")
740 740
741 741 self.assertIsInstance(res.error_before_exec, InputRejected)
742 742
743 743 def test__IPYTHON__():
744 744 # This shouldn't raise a NameError, that's all
745 745 __IPYTHON__
746 746
747 747
748 748 class DummyRepr(object):
749 749 def __repr__(self):
750 750 return "DummyRepr"
751 751
752 752 def _repr_html_(self):
753 753 return "<b>dummy</b>"
754 754
755 755 def _repr_javascript_(self):
756 756 return "console.log('hi');", {'key': 'value'}
757 757
758 758
759 759 def test_user_variables():
760 760 # enable all formatters
761 761 ip.display_formatter.active_types = ip.display_formatter.format_types
762 762
763 763 ip.user_ns['dummy'] = d = DummyRepr()
764 764 keys = {'dummy', 'doesnotexist'}
765 765 r = ip.user_expressions({ key:key for key in keys})
766 766
767 767 nt.assert_equal(keys, set(r.keys()))
768 768 dummy = r['dummy']
769 769 nt.assert_equal({'status', 'data', 'metadata'}, set(dummy.keys()))
770 770 nt.assert_equal(dummy['status'], 'ok')
771 771 data = dummy['data']
772 772 metadata = dummy['metadata']
773 773 nt.assert_equal(data.get('text/html'), d._repr_html_())
774 774 js, jsmd = d._repr_javascript_()
775 775 nt.assert_equal(data.get('application/javascript'), js)
776 776 nt.assert_equal(metadata.get('application/javascript'), jsmd)
777 777
778 778 dne = r['doesnotexist']
779 779 nt.assert_equal(dne['status'], 'error')
780 780 nt.assert_equal(dne['ename'], 'NameError')
781 781
782 782 # back to text only
783 783 ip.display_formatter.active_types = ['text/plain']
784 784
785 785 def test_user_expression():
786 786 # enable all formatters
787 787 ip.display_formatter.active_types = ip.display_formatter.format_types
788 788 query = {
789 789 'a' : '1 + 2',
790 790 'b' : '1/0',
791 791 }
792 792 r = ip.user_expressions(query)
793 793 import pprint
794 794 pprint.pprint(r)
795 795 nt.assert_equal(set(r.keys()), set(query.keys()))
796 796 a = r['a']
797 797 nt.assert_equal({'status', 'data', 'metadata'}, set(a.keys()))
798 798 nt.assert_equal(a['status'], 'ok')
799 799 data = a['data']
800 800 metadata = a['metadata']
801 801 nt.assert_equal(data.get('text/plain'), '3')
802 802
803 803 b = r['b']
804 804 nt.assert_equal(b['status'], 'error')
805 805 nt.assert_equal(b['ename'], 'ZeroDivisionError')
806 806
807 807 # back to text only
808 808 ip.display_formatter.active_types = ['text/plain']
809 809
810 810
811 811
812 812
813 813
814 814 class TestSyntaxErrorTransformer(unittest.TestCase):
815 815 """Check that SyntaxError raised by an input transformer is handled by run_cell()"""
816 816
817 817 class SyntaxErrorTransformer(InputTransformer):
818 818
819 819 def push(self, line):
820 820 pos = line.find('syntaxerror')
821 821 if pos >= 0:
822 822 e = SyntaxError('input contains "syntaxerror"')
823 823 e.text = line
824 824 e.offset = pos + 1
825 825 raise e
826 826 return line
827 827
828 828 def reset(self):
829 829 pass
830 830
831 831 def setUp(self):
832 832 self.transformer = TestSyntaxErrorTransformer.SyntaxErrorTransformer()
833 833 ip.input_splitter.python_line_transforms.append(self.transformer)
834 834 ip.input_transformer_manager.python_line_transforms.append(self.transformer)
835 835
836 836 def tearDown(self):
837 837 ip.input_splitter.python_line_transforms.remove(self.transformer)
838 838 ip.input_transformer_manager.python_line_transforms.remove(self.transformer)
839 839
840 840 def test_syntaxerror_input_transformer(self):
841 841 with tt.AssertPrints('1234'):
842 842 ip.run_cell('1234')
843 843 with tt.AssertPrints('SyntaxError: invalid syntax'):
844 844 ip.run_cell('1 2 3') # plain python syntax error
845 845 with tt.AssertPrints('SyntaxError: input contains "syntaxerror"'):
846 846 ip.run_cell('2345 # syntaxerror') # input transformer syntax error
847 847 with tt.AssertPrints('3456'):
848 848 ip.run_cell('3456')
849 849
850 850
851 851
852 852 def test_warning_suppression():
853 853 ip.run_cell("import warnings")
854 854 try:
855 855 with tt.AssertPrints("UserWarning: asdf", channel="stderr"):
856 856 ip.run_cell("warnings.warn('asdf')")
857 857 # Here's the real test -- if we run that again, we should get the
858 858 # warning again. Traditionally, each warning was only issued once per
859 859 # IPython session (approximately), even if the user typed in new and
860 860 # different code that should have also triggered the warning, leading
861 861 # to much confusion.
862 862 with tt.AssertPrints("UserWarning: asdf", channel="stderr"):
863 863 ip.run_cell("warnings.warn('asdf')")
864 864 finally:
865 865 ip.run_cell("del warnings")
866 866
867 867
868 868 def test_deprecation_warning():
869 869 ip.run_cell("""
870 870 import warnings
871 871 def wrn():
872 872 warnings.warn(
873 873 "I AM A WARNING",
874 874 DeprecationWarning
875 875 )
876 876 """)
877 877 try:
878 878 with tt.AssertPrints("I AM A WARNING", channel="stderr"):
879 879 ip.run_cell("wrn()")
880 880 finally:
881 881 ip.run_cell("del warnings")
882 882 ip.run_cell("del wrn")
883 883
884 884
885 885 class TestImportNoDeprecate(tt.TempFileMixin):
886 886
887 887 def setup(self):
888 888 """Make a valid python temp file."""
889 889 self.mktmp("""
890 890 import warnings
891 891 def wrn():
892 892 warnings.warn(
893 893 "I AM A WARNING",
894 894 DeprecationWarning
895 895 )
896 896 """)
897 897
898 898 def test_no_dep(self):
899 899 """
900 900 No deprecation warning should be raised from imported functions
901 901 """
902 902 ip.run_cell("from {} import wrn".format(self.fname))
903 903
904 904 with tt.AssertNotPrints("I AM A WARNING"):
905 905 ip.run_cell("wrn()")
906 906 ip.run_cell("del wrn")
@@ -1,987 +1,987 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for various magic functions.
3 3
4 4 Needs to be run by nose (to make ipython session available).
5 5 """
6 6
7 7 import io
8 8 import os
9 9 import sys
10 10 import warnings
11 11 from unittest import TestCase
12 12
13 13 try:
14 14 from importlib import invalidate_caches # Required from Python 3.3
15 15 except ImportError:
16 16 def invalidate_caches():
17 17 pass
18 18
19 19 import nose.tools as nt
20 20
21 21 from IPython import get_ipython
22 22 from IPython.core import magic
23 23 from IPython.core.error import UsageError
24 24 from IPython.core.magic import (Magics, magics_class, line_magic,
25 25 cell_magic,
26 26 register_line_magic, register_cell_magic)
27 27 from IPython.core.magics import execution, script, code
28 28 from IPython.testing import decorators as dec
29 29 from IPython.testing import tools as tt
30 30 from IPython.utils import py3compat
31 31 from IPython.utils.io import capture_output
32 32 from IPython.utils.tempdir import TemporaryDirectory
33 33 from IPython.utils.process import find_cmd
34 34
35 35 if py3compat.PY3:
36 36 from io import StringIO
37 37 else:
38 38 from StringIO import StringIO
39 39
40 40
41 41 _ip = get_ipython()
42 42
43 43 @magic.magics_class
44 44 class DummyMagics(magic.Magics): pass
45 45
46 46 def test_extract_code_ranges():
47 47 instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
48 48 expected = [(0, 1),
49 49 (2, 3),
50 50 (4, 6),
51 51 (6, 9),
52 52 (9, 14),
53 53 (16, None),
54 54 (None, 9),
55 55 (9, None),
56 56 (None, 13),
57 57 (None, None)]
58 58 actual = list(code.extract_code_ranges(instr))
59 59 nt.assert_equal(actual, expected)
60 60
61 61 def test_extract_symbols():
62 62 source = """import foo\na = 10\ndef b():\n return 42\n\n\nclass A: pass\n\n\n"""
63 63 symbols_args = ["a", "b", "A", "A,b", "A,a", "z"]
64 64 expected = [([], ['a']),
65 65 (["def b():\n return 42\n"], []),
66 66 (["class A: pass\n"], []),
67 67 (["class A: pass\n", "def b():\n return 42\n"], []),
68 68 (["class A: pass\n"], ['a']),
69 69 ([], ['z'])]
70 70 for symbols, exp in zip(symbols_args, expected):
71 71 nt.assert_equal(code.extract_symbols(source, symbols), exp)
72 72
73 73
74 74 def test_extract_symbols_raises_exception_with_non_python_code():
75 75 source = ("=begin A Ruby program :)=end\n"
76 76 "def hello\n"
77 77 "puts 'Hello world'\n"
78 78 "end")
79 79 with nt.assert_raises(SyntaxError):
80 80 code.extract_symbols(source, "hello")
81 81
82 82 def test_config():
83 83 """ test that config magic does not raise
84 84 can happen if Configurable init is moved too early into
85 85 Magics.__init__ as then a Config object will be registerd as a
86 86 magic.
87 87 """
88 88 ## should not raise.
89 89 _ip.magic('config')
90 90
91 91 def test_rehashx():
92 92 # clear up everything
93 93 _ip.alias_manager.clear_aliases()
94 94 del _ip.db['syscmdlist']
95 95
96 96 _ip.magic('rehashx')
97 97 # Practically ALL ipython development systems will have more than 10 aliases
98 98
99 99 nt.assert_true(len(_ip.alias_manager.aliases) > 10)
100 100 for name, cmd in _ip.alias_manager.aliases:
101 101 # we must strip dots from alias names
102 102 nt.assert_not_in('.', name)
103 103
104 104 # rehashx must fill up syscmdlist
105 105 scoms = _ip.db['syscmdlist']
106 106 nt.assert_true(len(scoms) > 10)
107 107
108 108
109 109 def test_magic_parse_options():
110 110 """Test that we don't mangle paths when parsing magic options."""
111 111 ip = get_ipython()
112 112 path = 'c:\\x'
113 113 m = DummyMagics(ip)
114 114 opts = m.parse_options('-f %s' % path,'f:')[0]
115 115 # argv splitting is os-dependent
116 116 if os.name == 'posix':
117 117 expected = 'c:x'
118 118 else:
119 119 expected = path
120 120 nt.assert_equal(opts['f'], expected)
121 121
122 122 def test_magic_parse_long_options():
123 123 """Magic.parse_options can handle --foo=bar long options"""
124 124 ip = get_ipython()
125 125 m = DummyMagics(ip)
126 126 opts, _ = m.parse_options('--foo --bar=bubble', 'a', 'foo', 'bar=')
127 127 nt.assert_in('foo', opts)
128 128 nt.assert_in('bar', opts)
129 129 nt.assert_equal(opts['bar'], "bubble")
130 130
131 131
132 132 @dec.skip_without('sqlite3')
133 133 def doctest_hist_f():
134 134 """Test %hist -f with temporary filename.
135 135
136 136 In [9]: import tempfile
137 137
138 138 In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
139 139
140 140 In [11]: %hist -nl -f $tfile 3
141 141
142 142 In [13]: import os; os.unlink(tfile)
143 143 """
144 144
145 145
146 146 @dec.skip_without('sqlite3')
147 147 def doctest_hist_r():
148 148 """Test %hist -r
149 149
150 150 XXX - This test is not recording the output correctly. For some reason, in
151 151 testing mode the raw history isn't getting populated. No idea why.
152 152 Disabling the output checking for now, though at least we do run it.
153 153
154 154 In [1]: 'hist' in _ip.lsmagic()
155 155 Out[1]: True
156 156
157 157 In [2]: x=1
158 158
159 159 In [3]: %hist -rl 2
160 160 x=1 # random
161 161 %hist -r 2
162 162 """
163 163
164 164
165 165 @dec.skip_without('sqlite3')
166 166 def doctest_hist_op():
167 167 """Test %hist -op
168 168
169 169 In [1]: class b(float):
170 170 ...: pass
171 171 ...:
172 172
173 173 In [2]: class s(object):
174 174 ...: def __str__(self):
175 175 ...: return 's'
176 176 ...:
177 177
178 178 In [3]:
179 179
180 180 In [4]: class r(b):
181 181 ...: def __repr__(self):
182 182 ...: return 'r'
183 183 ...:
184 184
185 185 In [5]: class sr(s,r): pass
186 186 ...:
187 187
188 188 In [6]:
189 189
190 190 In [7]: bb=b()
191 191
192 192 In [8]: ss=s()
193 193
194 194 In [9]: rr=r()
195 195
196 196 In [10]: ssrr=sr()
197 197
198 198 In [11]: 4.5
199 199 Out[11]: 4.5
200 200
201 201 In [12]: str(ss)
202 202 Out[12]: 's'
203 203
204 204 In [13]:
205 205
206 206 In [14]: %hist -op
207 207 >>> class b:
208 208 ... pass
209 209 ...
210 210 >>> class s(b):
211 211 ... def __str__(self):
212 212 ... return 's'
213 213 ...
214 214 >>>
215 215 >>> class r(b):
216 216 ... def __repr__(self):
217 217 ... return 'r'
218 218 ...
219 219 >>> class sr(s,r): pass
220 220 >>>
221 221 >>> bb=b()
222 222 >>> ss=s()
223 223 >>> rr=r()
224 224 >>> ssrr=sr()
225 225 >>> 4.5
226 226 4.5
227 227 >>> str(ss)
228 228 's'
229 229 >>>
230 230 """
231 231
232 232 def test_hist_pof():
233 233 ip = get_ipython()
234 234 ip.run_cell(u"1+2", store_history=True)
235 235 #raise Exception(ip.history_manager.session_number)
236 236 #raise Exception(list(ip.history_manager._get_range_session()))
237 237 with TemporaryDirectory() as td:
238 238 tf = os.path.join(td, 'hist.py')
239 239 ip.run_line_magic('history', '-pof %s' % tf)
240 240 assert os.path.isfile(tf)
241 241
242 242
243 243 @dec.skip_without('sqlite3')
244 244 def test_macro():
245 245 ip = get_ipython()
246 246 ip.history_manager.reset() # Clear any existing history.
247 247 cmds = ["a=1", "def b():\n return a**2", "print(a,b())"]
248 248 for i, cmd in enumerate(cmds, start=1):
249 249 ip.history_manager.store_inputs(i, cmd)
250 250 ip.magic("macro test 1-3")
251 251 nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
252 252
253 253 # List macros
254 254 nt.assert_in("test", ip.magic("macro"))
255 255
256 256
257 257 @dec.skip_without('sqlite3')
258 258 def test_macro_run():
259 259 """Test that we can run a multi-line macro successfully."""
260 260 ip = get_ipython()
261 261 ip.history_manager.reset()
262 262 cmds = ["a=10", "a+=1", py3compat.doctest_refactor_print("print a"),
263 263 "%macro test 2-3"]
264 264 for cmd in cmds:
265 265 ip.run_cell(cmd, store_history=True)
266 266 nt.assert_equal(ip.user_ns["test"].value,
267 267 py3compat.doctest_refactor_print("a+=1\nprint a\n"))
268 268 with tt.AssertPrints("12"):
269 269 ip.run_cell("test")
270 270 with tt.AssertPrints("13"):
271 271 ip.run_cell("test")
272 272
273 273
274 274 def test_magic_magic():
275 275 """Test %magic"""
276 276 ip = get_ipython()
277 277 with capture_output() as captured:
278 278 ip.magic("magic")
279 279
280 280 stdout = captured.stdout
281 281 nt.assert_in('%magic', stdout)
282 282 nt.assert_in('IPython', stdout)
283 283 nt.assert_in('Available', stdout)
284 284
285 285
286 286 @dec.skipif_not_numpy
287 287 def test_numpy_reset_array_undec():
288 288 "Test '%reset array' functionality"
289 289 _ip.ex('import numpy as np')
290 290 _ip.ex('a = np.empty(2)')
291 291 nt.assert_in('a', _ip.user_ns)
292 292 _ip.magic('reset -f array')
293 293 nt.assert_not_in('a', _ip.user_ns)
294 294
295 295 def test_reset_out():
296 296 "Test '%reset out' magic"
297 297 _ip.run_cell("parrot = 'dead'", store_history=True)
298 298 # test '%reset -f out', make an Out prompt
299 299 _ip.run_cell("parrot", store_history=True)
300 300 nt.assert_true('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
301 301 _ip.magic('reset -f out')
302 302 nt.assert_false('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
303 303 nt.assert_equal(len(_ip.user_ns['Out']), 0)
304 304
305 305 def test_reset_in():
306 306 "Test '%reset in' magic"
307 307 # test '%reset -f in'
308 308 _ip.run_cell("parrot", store_history=True)
309 309 nt.assert_true('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
310 310 _ip.magic('%reset -f in')
311 311 nt.assert_false('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
312 312 nt.assert_equal(len(set(_ip.user_ns['In'])), 1)
313 313
314 314 def test_reset_dhist():
315 315 "Test '%reset dhist' magic"
316 316 _ip.run_cell("tmp = [d for d in _dh]") # copy before clearing
317 317 _ip.magic('cd ' + os.path.dirname(nt.__file__))
318 318 _ip.magic('cd -')
319 319 nt.assert_true(len(_ip.user_ns['_dh']) > 0)
320 320 _ip.magic('reset -f dhist')
321 321 nt.assert_equal(len(_ip.user_ns['_dh']), 0)
322 322 _ip.run_cell("_dh = [d for d in tmp]") #restore
323 323
324 324 def test_reset_in_length():
325 325 "Test that '%reset in' preserves In[] length"
326 326 _ip.run_cell("print 'foo'")
327 327 _ip.run_cell("reset -f in")
328 328 nt.assert_equal(len(_ip.user_ns['In']), _ip.displayhook.prompt_count+1)
329 329
330 330 def test_tb_syntaxerror():
331 331 """test %tb after a SyntaxError"""
332 332 ip = get_ipython()
333 333 ip.run_cell("for")
334 334
335 335 # trap and validate stdout
336 336 save_stdout = sys.stdout
337 337 try:
338 338 sys.stdout = StringIO()
339 339 ip.run_cell("%tb")
340 340 out = sys.stdout.getvalue()
341 341 finally:
342 342 sys.stdout = save_stdout
343 343 # trim output, and only check the last line
344 344 last_line = out.rstrip().splitlines()[-1].strip()
345 345 nt.assert_equal(last_line, "SyntaxError: invalid syntax")
346 346
347 347
348 348 def test_time():
349 349 ip = get_ipython()
350 350
351 351 with tt.AssertPrints("Wall time: "):
352 352 ip.run_cell("%time None")
353 353
354 354 ip.run_cell("def f(kmjy):\n"
355 355 " %time print (2*kmjy)")
356 356
357 357 with tt.AssertPrints("Wall time: "):
358 358 with tt.AssertPrints("hihi", suppress=False):
359 359 ip.run_cell("f('hi')")
360 360
361 361
362 362 @dec.skip_win32
363 363 def test_time2():
364 364 ip = get_ipython()
365 365
366 366 with tt.AssertPrints("CPU times: user "):
367 367 ip.run_cell("%time None")
368 368
369 369 def test_time3():
370 370 """Erroneous magic function calls, issue gh-3334"""
371 371 ip = get_ipython()
372 372 ip.user_ns.pop('run', None)
373 373
374 374 with tt.AssertNotPrints("not found", channel='stderr'):
375 375 ip.run_cell("%%time\n"
376 376 "run = 0\n"
377 377 "run += 1")
378 378
379 379 def test_doctest_mode():
380 380 "Toggle doctest_mode twice, it should be a no-op and run without error"
381 381 _ip.magic('doctest_mode')
382 382 _ip.magic('doctest_mode')
383 383
384 384
385 385 def test_parse_options():
386 386 """Tests for basic options parsing in magics."""
387 387 # These are only the most minimal of tests, more should be added later. At
388 388 # the very least we check that basic text/unicode calls work OK.
389 389 m = DummyMagics(_ip)
390 390 nt.assert_equal(m.parse_options('foo', '')[1], 'foo')
391 391 nt.assert_equal(m.parse_options(u'foo', '')[1], u'foo')
392 392
393 393
394 394 def test_dirops():
395 395 """Test various directory handling operations."""
396 # curpath = lambda :os.path.splitdrive(py3compat.getcwd())[1].replace('\\','/')
397 curpath = py3compat.getcwd
398 startdir = py3compat.getcwd()
396 # curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
397 curpath = os.getcwd
398 startdir = os.getcwd()
399 399 ipdir = os.path.realpath(_ip.ipython_dir)
400 400 try:
401 401 _ip.magic('cd "%s"' % ipdir)
402 402 nt.assert_equal(curpath(), ipdir)
403 403 _ip.magic('cd -')
404 404 nt.assert_equal(curpath(), startdir)
405 405 _ip.magic('pushd "%s"' % ipdir)
406 406 nt.assert_equal(curpath(), ipdir)
407 407 _ip.magic('popd')
408 408 nt.assert_equal(curpath(), startdir)
409 409 finally:
410 410 os.chdir(startdir)
411 411
412 412
413 413 def test_xmode():
414 414 # Calling xmode three times should be a no-op
415 415 xmode = _ip.InteractiveTB.mode
416 416 for i in range(3):
417 417 _ip.magic("xmode")
418 418 nt.assert_equal(_ip.InteractiveTB.mode, xmode)
419 419
420 420 def test_reset_hard():
421 421 monitor = []
422 422 class A(object):
423 423 def __del__(self):
424 424 monitor.append(1)
425 425 def __repr__(self):
426 426 return "<A instance>"
427 427
428 428 _ip.user_ns["a"] = A()
429 429 _ip.run_cell("a")
430 430
431 431 nt.assert_equal(monitor, [])
432 432 _ip.magic("reset -f")
433 433 nt.assert_equal(monitor, [1])
434 434
435 435 class TestXdel(tt.TempFileMixin):
436 436 def test_xdel(self):
437 437 """Test that references from %run are cleared by xdel."""
438 438 src = ("class A(object):\n"
439 439 " monitor = []\n"
440 440 " def __del__(self):\n"
441 441 " self.monitor.append(1)\n"
442 442 "a = A()\n")
443 443 self.mktmp(src)
444 444 # %run creates some hidden references...
445 445 _ip.magic("run %s" % self.fname)
446 446 # ... as does the displayhook.
447 447 _ip.run_cell("a")
448 448
449 449 monitor = _ip.user_ns["A"].monitor
450 450 nt.assert_equal(monitor, [])
451 451
452 452 _ip.magic("xdel a")
453 453
454 454 # Check that a's __del__ method has been called.
455 455 nt.assert_equal(monitor, [1])
456 456
457 457 def doctest_who():
458 458 """doctest for %who
459 459
460 460 In [1]: %reset -f
461 461
462 462 In [2]: alpha = 123
463 463
464 464 In [3]: beta = 'beta'
465 465
466 466 In [4]: %who int
467 467 alpha
468 468
469 469 In [5]: %who str
470 470 beta
471 471
472 472 In [6]: %whos
473 473 Variable Type Data/Info
474 474 ----------------------------
475 475 alpha int 123
476 476 beta str beta
477 477
478 478 In [7]: %who_ls
479 479 Out[7]: ['alpha', 'beta']
480 480 """
481 481
482 482 def test_whos():
483 483 """Check that whos is protected against objects where repr() fails."""
484 484 class A(object):
485 485 def __repr__(self):
486 486 raise Exception()
487 487 _ip.user_ns['a'] = A()
488 488 _ip.magic("whos")
489 489
490 490 @py3compat.u_format
491 491 def doctest_precision():
492 492 """doctest for %precision
493 493
494 494 In [1]: f = get_ipython().display_formatter.formatters['text/plain']
495 495
496 496 In [2]: %precision 5
497 497 Out[2]: {u}'%.5f'
498 498
499 499 In [3]: f.float_format
500 500 Out[3]: {u}'%.5f'
501 501
502 502 In [4]: %precision %e
503 503 Out[4]: {u}'%e'
504 504
505 505 In [5]: f(3.1415927)
506 506 Out[5]: {u}'3.141593e+00'
507 507 """
508 508
509 509 def test_psearch():
510 510 with tt.AssertPrints("dict.fromkeys"):
511 511 _ip.run_cell("dict.fr*?")
512 512
513 513 def test_timeit_shlex():
514 514 """test shlex issues with timeit (#1109)"""
515 515 _ip.ex("def f(*a,**kw): pass")
516 516 _ip.magic('timeit -n1 "this is a bug".count(" ")')
517 517 _ip.magic('timeit -r1 -n1 f(" ", 1)')
518 518 _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
519 519 _ip.magic('timeit -r1 -n1 ("a " + "b")')
520 520 _ip.magic('timeit -r1 -n1 f("a " + "b")')
521 521 _ip.magic('timeit -r1 -n1 f("a " + "b ")')
522 522
523 523
524 524 def test_timeit_arguments():
525 525 "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
526 526 _ip.magic("timeit ('#')")
527 527
528 528
529 529 def test_timeit_special_syntax():
530 530 "Test %%timeit with IPython special syntax"
531 531 @register_line_magic
532 532 def lmagic(line):
533 533 ip = get_ipython()
534 534 ip.user_ns['lmagic_out'] = line
535 535
536 536 # line mode test
537 537 _ip.run_line_magic('timeit', '-n1 -r1 %lmagic my line')
538 538 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
539 539 # cell mode test
540 540 _ip.run_cell_magic('timeit', '-n1 -r1', '%lmagic my line2')
541 541 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
542 542
543 543 def test_timeit_return():
544 544 """
545 545 test wether timeit -o return object
546 546 """
547 547
548 548 res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
549 549 assert(res is not None)
550 550
551 551 def test_timeit_quiet():
552 552 """
553 553 test quiet option of timeit magic
554 554 """
555 555 with tt.AssertNotPrints("loops"):
556 556 _ip.run_cell("%timeit -n1 -r1 -q 1")
557 557
558 558 def test_timeit_return_quiet():
559 559 with tt.AssertNotPrints("loops"):
560 560 res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
561 561 assert (res is not None)
562 562
563 563 @dec.skipif(execution.profile is None)
564 564 def test_prun_special_syntax():
565 565 "Test %%prun with IPython special syntax"
566 566 @register_line_magic
567 567 def lmagic(line):
568 568 ip = get_ipython()
569 569 ip.user_ns['lmagic_out'] = line
570 570
571 571 # line mode test
572 572 _ip.run_line_magic('prun', '-q %lmagic my line')
573 573 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
574 574 # cell mode test
575 575 _ip.run_cell_magic('prun', '-q', '%lmagic my line2')
576 576 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
577 577
578 578 @dec.skipif(execution.profile is None)
579 579 def test_prun_quotes():
580 580 "Test that prun does not clobber string escapes (GH #1302)"
581 581 _ip.magic(r"prun -q x = '\t'")
582 582 nt.assert_equal(_ip.user_ns['x'], '\t')
583 583
584 584 def test_extension():
585 585 # Debugging information for failures of this test
586 586 print('sys.path:')
587 587 for p in sys.path:
588 588 print(' ', p)
589 589 print('CWD', os.getcwd())
590 590
591 591 nt.assert_raises(ImportError, _ip.magic, "load_ext daft_extension")
592 592 daft_path = os.path.join(os.path.dirname(__file__), "daft_extension")
593 593 sys.path.insert(0, daft_path)
594 594 try:
595 595 _ip.user_ns.pop('arq', None)
596 596 invalidate_caches() # Clear import caches
597 597 _ip.magic("load_ext daft_extension")
598 598 nt.assert_equal(_ip.user_ns['arq'], 185)
599 599 _ip.magic("unload_ext daft_extension")
600 600 assert 'arq' not in _ip.user_ns
601 601 finally:
602 602 sys.path.remove(daft_path)
603 603
604 604
605 605 def test_notebook_export_json():
606 606 _ip = get_ipython()
607 607 _ip.history_manager.reset() # Clear any existing history.
608 608 cmds = [u"a=1", u"def b():\n return a**2", u"print('noΓ«l, Γ©tΓ©', b())"]
609 609 for i, cmd in enumerate(cmds, start=1):
610 610 _ip.history_manager.store_inputs(i, cmd)
611 611 with TemporaryDirectory() as td:
612 612 outfile = os.path.join(td, "nb.ipynb")
613 613 _ip.magic("notebook -e %s" % outfile)
614 614
615 615
616 616 class TestEnv(TestCase):
617 617
618 618 def test_env(self):
619 619 env = _ip.magic("env")
620 620 self.assertTrue(isinstance(env, dict))
621 621
622 622 def test_env_get_set_simple(self):
623 623 env = _ip.magic("env var val1")
624 624 self.assertEqual(env, None)
625 625 self.assertEqual(os.environ['var'], 'val1')
626 626 self.assertEqual(_ip.magic("env var"), 'val1')
627 627 env = _ip.magic("env var=val2")
628 628 self.assertEqual(env, None)
629 629 self.assertEqual(os.environ['var'], 'val2')
630 630
631 631 def test_env_get_set_complex(self):
632 632 env = _ip.magic("env var 'val1 '' 'val2")
633 633 self.assertEqual(env, None)
634 634 self.assertEqual(os.environ['var'], "'val1 '' 'val2")
635 635 self.assertEqual(_ip.magic("env var"), "'val1 '' 'val2")
636 636 env = _ip.magic('env var=val2 val3="val4')
637 637 self.assertEqual(env, None)
638 638 self.assertEqual(os.environ['var'], 'val2 val3="val4')
639 639
640 640 def test_env_set_bad_input(self):
641 641 self.assertRaises(UsageError, lambda: _ip.magic("set_env var"))
642 642
643 643 def test_env_set_whitespace(self):
644 644 self.assertRaises(UsageError, lambda: _ip.magic("env var A=B"))
645 645
646 646
647 647 class CellMagicTestCase(TestCase):
648 648
649 649 def check_ident(self, magic):
650 650 # Manually called, we get the result
651 651 out = _ip.run_cell_magic(magic, 'a', 'b')
652 652 nt.assert_equal(out, ('a','b'))
653 653 # Via run_cell, it goes into the user's namespace via displayhook
654 654 _ip.run_cell('%%' + magic +' c\nd')
655 655 nt.assert_equal(_ip.user_ns['_'], ('c','d'))
656 656
657 657 def test_cell_magic_func_deco(self):
658 658 "Cell magic using simple decorator"
659 659 @register_cell_magic
660 660 def cellm(line, cell):
661 661 return line, cell
662 662
663 663 self.check_ident('cellm')
664 664
665 665 def test_cell_magic_reg(self):
666 666 "Cell magic manually registered"
667 667 def cellm(line, cell):
668 668 return line, cell
669 669
670 670 _ip.register_magic_function(cellm, 'cell', 'cellm2')
671 671 self.check_ident('cellm2')
672 672
673 673 def test_cell_magic_class(self):
674 674 "Cell magics declared via a class"
675 675 @magics_class
676 676 class MyMagics(Magics):
677 677
678 678 @cell_magic
679 679 def cellm3(self, line, cell):
680 680 return line, cell
681 681
682 682 _ip.register_magics(MyMagics)
683 683 self.check_ident('cellm3')
684 684
685 685 def test_cell_magic_class2(self):
686 686 "Cell magics declared via a class, #2"
687 687 @magics_class
688 688 class MyMagics2(Magics):
689 689
690 690 @cell_magic('cellm4')
691 691 def cellm33(self, line, cell):
692 692 return line, cell
693 693
694 694 _ip.register_magics(MyMagics2)
695 695 self.check_ident('cellm4')
696 696 # Check that nothing is registered as 'cellm33'
697 697 c33 = _ip.find_cell_magic('cellm33')
698 698 nt.assert_equal(c33, None)
699 699
700 700 def test_file():
701 701 """Basic %%file"""
702 702 ip = get_ipython()
703 703 with TemporaryDirectory() as td:
704 704 fname = os.path.join(td, 'file1')
705 705 ip.run_cell_magic("file", fname, u'\n'.join([
706 706 'line1',
707 707 'line2',
708 708 ]))
709 709 with open(fname) as f:
710 710 s = f.read()
711 711 nt.assert_in('line1\n', s)
712 712 nt.assert_in('line2', s)
713 713
714 714 def test_file_var_expand():
715 715 """%%file $filename"""
716 716 ip = get_ipython()
717 717 with TemporaryDirectory() as td:
718 718 fname = os.path.join(td, 'file1')
719 719 ip.user_ns['filename'] = fname
720 720 ip.run_cell_magic("file", '$filename', u'\n'.join([
721 721 'line1',
722 722 'line2',
723 723 ]))
724 724 with open(fname) as f:
725 725 s = f.read()
726 726 nt.assert_in('line1\n', s)
727 727 nt.assert_in('line2', s)
728 728
729 729 def test_file_unicode():
730 730 """%%file with unicode cell"""
731 731 ip = get_ipython()
732 732 with TemporaryDirectory() as td:
733 733 fname = os.path.join(td, 'file1')
734 734 ip.run_cell_magic("file", fname, u'\n'.join([
735 735 u'linΓ©1',
736 736 u'linΓ©2',
737 737 ]))
738 738 with io.open(fname, encoding='utf-8') as f:
739 739 s = f.read()
740 740 nt.assert_in(u'linΓ©1\n', s)
741 741 nt.assert_in(u'linΓ©2', s)
742 742
743 743 def test_file_amend():
744 744 """%%file -a amends files"""
745 745 ip = get_ipython()
746 746 with TemporaryDirectory() as td:
747 747 fname = os.path.join(td, 'file2')
748 748 ip.run_cell_magic("file", fname, u'\n'.join([
749 749 'line1',
750 750 'line2',
751 751 ]))
752 752 ip.run_cell_magic("file", "-a %s" % fname, u'\n'.join([
753 753 'line3',
754 754 'line4',
755 755 ]))
756 756 with open(fname) as f:
757 757 s = f.read()
758 758 nt.assert_in('line1\n', s)
759 759 nt.assert_in('line3\n', s)
760 760
761 761
762 762 def test_script_config():
763 763 ip = get_ipython()
764 764 ip.config.ScriptMagics.script_magics = ['whoda']
765 765 sm = script.ScriptMagics(shell=ip)
766 766 nt.assert_in('whoda', sm.magics['cell'])
767 767
768 768 @dec.skip_win32
769 769 def test_script_out():
770 770 ip = get_ipython()
771 771 ip.run_cell_magic("script", "--out output sh", "echo 'hi'")
772 772 nt.assert_equal(ip.user_ns['output'], 'hi\n')
773 773
774 774 @dec.skip_win32
775 775 def test_script_err():
776 776 ip = get_ipython()
777 777 ip.run_cell_magic("script", "--err error sh", "echo 'hello' >&2")
778 778 nt.assert_equal(ip.user_ns['error'], 'hello\n')
779 779
780 780 @dec.skip_win32
781 781 def test_script_out_err():
782 782 ip = get_ipython()
783 783 ip.run_cell_magic("script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2")
784 784 nt.assert_equal(ip.user_ns['output'], 'hi\n')
785 785 nt.assert_equal(ip.user_ns['error'], 'hello\n')
786 786
787 787 @dec.skip_win32
788 788 def test_script_bg_out():
789 789 ip = get_ipython()
790 790 ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'")
791 791 nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
792 792
793 793 @dec.skip_win32
794 794 def test_script_bg_err():
795 795 ip = get_ipython()
796 796 ip.run_cell_magic("script", "--bg --err error sh", "echo 'hello' >&2")
797 797 nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
798 798
799 799 @dec.skip_win32
800 800 def test_script_bg_out_err():
801 801 ip = get_ipython()
802 802 ip.run_cell_magic("script", "--bg --out output --err error sh", "echo 'hi'\necho 'hello' >&2")
803 803 nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
804 804 nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
805 805
806 806 def test_script_defaults():
807 807 ip = get_ipython()
808 808 for cmd in ['sh', 'bash', 'perl', 'ruby']:
809 809 try:
810 810 find_cmd(cmd)
811 811 except Exception:
812 812 pass
813 813 else:
814 814 nt.assert_in(cmd, ip.magics_manager.magics['cell'])
815 815
816 816
817 817 @magics_class
818 818 class FooFoo(Magics):
819 819 """class with both %foo and %%foo magics"""
820 820 @line_magic('foo')
821 821 def line_foo(self, line):
822 822 "I am line foo"
823 823 pass
824 824
825 825 @cell_magic("foo")
826 826 def cell_foo(self, line, cell):
827 827 "I am cell foo, not line foo"
828 828 pass
829 829
830 830 def test_line_cell_info():
831 831 """%%foo and %foo magics are distinguishable to inspect"""
832 832 ip = get_ipython()
833 833 ip.magics_manager.register(FooFoo)
834 834 oinfo = ip.object_inspect('foo')
835 835 nt.assert_true(oinfo['found'])
836 836 nt.assert_true(oinfo['ismagic'])
837 837
838 838 oinfo = ip.object_inspect('%%foo')
839 839 nt.assert_true(oinfo['found'])
840 840 nt.assert_true(oinfo['ismagic'])
841 841 nt.assert_equal(oinfo['docstring'], FooFoo.cell_foo.__doc__)
842 842
843 843 oinfo = ip.object_inspect('%foo')
844 844 nt.assert_true(oinfo['found'])
845 845 nt.assert_true(oinfo['ismagic'])
846 846 nt.assert_equal(oinfo['docstring'], FooFoo.line_foo.__doc__)
847 847
848 848 def test_multiple_magics():
849 849 ip = get_ipython()
850 850 foo1 = FooFoo(ip)
851 851 foo2 = FooFoo(ip)
852 852 mm = ip.magics_manager
853 853 mm.register(foo1)
854 854 nt.assert_true(mm.magics['line']['foo'].__self__ is foo1)
855 855 mm.register(foo2)
856 856 nt.assert_true(mm.magics['line']['foo'].__self__ is foo2)
857 857
858 858 def test_alias_magic():
859 859 """Test %alias_magic."""
860 860 ip = get_ipython()
861 861 mm = ip.magics_manager
862 862
863 863 # Basic operation: both cell and line magics are created, if possible.
864 864 ip.run_line_magic('alias_magic', 'timeit_alias timeit')
865 865 nt.assert_in('timeit_alias', mm.magics['line'])
866 866 nt.assert_in('timeit_alias', mm.magics['cell'])
867 867
868 868 # --cell is specified, line magic not created.
869 869 ip.run_line_magic('alias_magic', '--cell timeit_cell_alias timeit')
870 870 nt.assert_not_in('timeit_cell_alias', mm.magics['line'])
871 871 nt.assert_in('timeit_cell_alias', mm.magics['cell'])
872 872
873 873 # Test that line alias is created successfully.
874 874 ip.run_line_magic('alias_magic', '--line env_alias env')
875 875 nt.assert_equal(ip.run_line_magic('env', ''),
876 876 ip.run_line_magic('env_alias', ''))
877 877
878 878 def test_save():
879 879 """Test %save."""
880 880 ip = get_ipython()
881 881 ip.history_manager.reset() # Clear any existing history.
882 882 cmds = [u"a=1", u"def b():\n return a**2", u"print(a, b())"]
883 883 for i, cmd in enumerate(cmds, start=1):
884 884 ip.history_manager.store_inputs(i, cmd)
885 885 with TemporaryDirectory() as tmpdir:
886 886 file = os.path.join(tmpdir, "testsave.py")
887 887 ip.run_line_magic("save", "%s 1-10" % file)
888 888 with open(file) as f:
889 889 content = f.read()
890 890 nt.assert_equal(content.count(cmds[0]), 1)
891 891 nt.assert_in('coding: utf-8', content)
892 892 ip.run_line_magic("save", "-a %s 1-10" % file)
893 893 with open(file) as f:
894 894 content = f.read()
895 895 nt.assert_equal(content.count(cmds[0]), 2)
896 896 nt.assert_in('coding: utf-8', content)
897 897
898 898
899 899 def test_store():
900 900 """Test %store."""
901 901 ip = get_ipython()
902 902 ip.run_line_magic('load_ext', 'storemagic')
903 903
904 904 # make sure the storage is empty
905 905 ip.run_line_magic('store', '-z')
906 906 ip.user_ns['var'] = 42
907 907 ip.run_line_magic('store', 'var')
908 908 ip.user_ns['var'] = 39
909 909 ip.run_line_magic('store', '-r')
910 910 nt.assert_equal(ip.user_ns['var'], 42)
911 911
912 912 ip.run_line_magic('store', '-d var')
913 913 ip.user_ns['var'] = 39
914 914 ip.run_line_magic('store' , '-r')
915 915 nt.assert_equal(ip.user_ns['var'], 39)
916 916
917 917
918 918 def _run_edit_test(arg_s, exp_filename=None,
919 919 exp_lineno=-1,
920 920 exp_contents=None,
921 921 exp_is_temp=None):
922 922 ip = get_ipython()
923 923 M = code.CodeMagics(ip)
924 924 last_call = ['','']
925 925 opts,args = M.parse_options(arg_s,'prxn:')
926 926 filename, lineno, is_temp = M._find_edit_target(ip, args, opts, last_call)
927 927
928 928 if exp_filename is not None:
929 929 nt.assert_equal(exp_filename, filename)
930 930 if exp_contents is not None:
931 931 with io.open(filename, 'r', encoding='utf-8') as f:
932 932 contents = f.read()
933 933 nt.assert_equal(exp_contents, contents)
934 934 if exp_lineno != -1:
935 935 nt.assert_equal(exp_lineno, lineno)
936 936 if exp_is_temp is not None:
937 937 nt.assert_equal(exp_is_temp, is_temp)
938 938
939 939
940 940 def test_edit_interactive():
941 941 """%edit on interactively defined objects"""
942 942 ip = get_ipython()
943 943 n = ip.execution_count
944 944 ip.run_cell(u"def foo(): return 1", store_history=True)
945 945
946 946 try:
947 947 _run_edit_test("foo")
948 948 except code.InteractivelyDefined as e:
949 949 nt.assert_equal(e.index, n)
950 950 else:
951 951 raise AssertionError("Should have raised InteractivelyDefined")
952 952
953 953
954 954 def test_edit_cell():
955 955 """%edit [cell id]"""
956 956 ip = get_ipython()
957 957
958 958 ip.run_cell(u"def foo(): return 1", store_history=True)
959 959
960 960 # test
961 961 _run_edit_test("1", exp_contents=ip.user_ns['In'][1], exp_is_temp=True)
962 962
963 963 def test_bookmark():
964 964 ip = get_ipython()
965 965 ip.run_line_magic('bookmark', 'bmname')
966 966 with tt.AssertPrints('bmname'):
967 967 ip.run_line_magic('bookmark', '-l')
968 968 ip.run_line_magic('bookmark', '-d bmname')
969 969
970 970 def test_ls_magic():
971 971 ip = get_ipython()
972 972 json_formatter = ip.display_formatter.formatters['application/json']
973 973 json_formatter.enabled = True
974 974 lsmagic = ip.magic('lsmagic')
975 975 with warnings.catch_warnings(record=True) as w:
976 976 j = json_formatter(lsmagic)
977 977 nt.assert_equal(sorted(j), ['cell', 'line'])
978 978 nt.assert_equal(w, []) # no warnings
979 979
980 980 def test_strip_initial_indent():
981 981 def sii(s):
982 982 lines = s.splitlines()
983 983 return '\n'.join(code.strip_initial_indent(lines))
984 984
985 985 nt.assert_equal(sii(" a = 1\nb = 2"), "a = 1\nb = 2")
986 986 nt.assert_equal(sii(" a\n b\nc"), "a\n b\nc")
987 987 nt.assert_equal(sii("a\n b"), "a\n b")
@@ -1,514 +1,514 b''
1 1 # encoding: utf-8
2 2 """Tests for code execution (%run and related), which is particularly tricky.
3 3
4 4 Because of how %run manages namespaces, and the fact that we are trying here to
5 5 verify subtle object deletion and reference counting issues, the %run tests
6 6 will be kept in this separate file. This makes it easier to aggregate in one
7 7 place the tricks needed to handle it; most other magics are much easier to test
8 8 and we do so in a common test_magic file.
9 9 """
10 10
11 11 # Copyright (c) IPython Development Team.
12 12 # Distributed under the terms of the Modified BSD License.
13 13
14 14
15 15
16 16 import functools
17 17 import os
18 18 from os.path import join as pjoin
19 19 import random
20 20 import sys
21 21 import textwrap
22 22 import unittest
23 23
24 24 try:
25 25 from unittest.mock import patch
26 26 except ImportError:
27 27 from mock import patch
28 28
29 29 import nose.tools as nt
30 30 from nose import SkipTest
31 31
32 32 from IPython.testing import decorators as dec
33 33 from IPython.testing import tools as tt
34 34 from IPython.utils import py3compat
35 35 from IPython.utils.io import capture_output
36 36 from IPython.utils.tempdir import TemporaryDirectory
37 37 from IPython.core import debugger
38 38
39 39
40 40 def doctest_refbug():
41 41 """Very nasty problem with references held by multiple runs of a script.
42 42 See: https://github.com/ipython/ipython/issues/141
43 43
44 44 In [1]: _ip.clear_main_mod_cache()
45 45 # random
46 46
47 47 In [2]: %run refbug
48 48
49 49 In [3]: call_f()
50 50 lowercased: hello
51 51
52 52 In [4]: %run refbug
53 53
54 54 In [5]: call_f()
55 55 lowercased: hello
56 56 lowercased: hello
57 57 """
58 58
59 59
60 60 def doctest_run_builtins():
61 61 r"""Check that %run doesn't damage __builtins__.
62 62
63 63 In [1]: import tempfile
64 64
65 65 In [2]: bid1 = id(__builtins__)
66 66
67 67 In [3]: fname = tempfile.mkstemp('.py')[1]
68 68
69 69 In [3]: f = open(fname,'w')
70 70
71 71 In [4]: dummy= f.write('pass\n')
72 72
73 73 In [5]: f.flush()
74 74
75 75 In [6]: t1 = type(__builtins__)
76 76
77 77 In [7]: %run $fname
78 78
79 79 In [7]: f.close()
80 80
81 81 In [8]: bid2 = id(__builtins__)
82 82
83 83 In [9]: t2 = type(__builtins__)
84 84
85 85 In [10]: t1 == t2
86 86 Out[10]: True
87 87
88 88 In [10]: bid1 == bid2
89 89 Out[10]: True
90 90
91 91 In [12]: try:
92 92 ....: os.unlink(fname)
93 93 ....: except:
94 94 ....: pass
95 95 ....:
96 96 """
97 97
98 98
99 99 def doctest_run_option_parser():
100 100 r"""Test option parser in %run.
101 101
102 102 In [1]: %run print_argv.py
103 103 []
104 104
105 105 In [2]: %run print_argv.py print*.py
106 106 ['print_argv.py']
107 107
108 108 In [3]: %run -G print_argv.py print*.py
109 109 ['print*.py']
110 110
111 111 """
112 112
113 113
114 114 @dec.skip_win32
115 115 def doctest_run_option_parser_for_posix():
116 116 r"""Test option parser in %run (Linux/OSX specific).
117 117
118 118 You need double quote to escape glob in POSIX systems:
119 119
120 120 In [1]: %run print_argv.py print\\*.py
121 121 ['print*.py']
122 122
123 123 You can't use quote to escape glob in POSIX systems:
124 124
125 125 In [2]: %run print_argv.py 'print*.py'
126 126 ['print_argv.py']
127 127
128 128 """
129 129
130 130
131 131 @dec.skip_if_not_win32
132 132 def doctest_run_option_parser_for_windows():
133 133 r"""Test option parser in %run (Windows specific).
134 134
135 135 In Windows, you can't escape ``*` `by backslash:
136 136
137 137 In [1]: %run print_argv.py print\\*.py
138 138 ['print\\*.py']
139 139
140 140 You can use quote to escape glob:
141 141
142 142 In [2]: %run print_argv.py 'print*.py'
143 143 ['print*.py']
144 144
145 145 """
146 146
147 147
148 148 @py3compat.doctest_refactor_print
149 149 def doctest_reset_del():
150 150 """Test that resetting doesn't cause errors in __del__ methods.
151 151
152 152 In [2]: class A(object):
153 153 ...: def __del__(self):
154 154 ...: print str("Hi")
155 155 ...:
156 156
157 157 In [3]: a = A()
158 158
159 159 In [4]: get_ipython().reset()
160 160 Hi
161 161
162 162 In [5]: 1+1
163 163 Out[5]: 2
164 164 """
165 165
166 166 # For some tests, it will be handy to organize them in a class with a common
167 167 # setup that makes a temp file
168 168
169 169 class TestMagicRunPass(tt.TempFileMixin):
170 170
171 171 def setup(self):
172 172 """Make a valid python temp file."""
173 173 self.mktmp('pass\n')
174 174
175 175 def run_tmpfile(self):
176 176 _ip = get_ipython()
177 177 # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
178 178 # See below and ticket https://bugs.launchpad.net/bugs/366353
179 179 _ip.magic('run %s' % self.fname)
180 180
181 181 def run_tmpfile_p(self):
182 182 _ip = get_ipython()
183 183 # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
184 184 # See below and ticket https://bugs.launchpad.net/bugs/366353
185 185 _ip.magic('run -p %s' % self.fname)
186 186
187 187 def test_builtins_id(self):
188 188 """Check that %run doesn't damage __builtins__ """
189 189 _ip = get_ipython()
190 190 # Test that the id of __builtins__ is not modified by %run
191 191 bid1 = id(_ip.user_ns['__builtins__'])
192 192 self.run_tmpfile()
193 193 bid2 = id(_ip.user_ns['__builtins__'])
194 194 nt.assert_equal(bid1, bid2)
195 195
196 196 def test_builtins_type(self):
197 197 """Check that the type of __builtins__ doesn't change with %run.
198 198
199 199 However, the above could pass if __builtins__ was already modified to
200 200 be a dict (it should be a module) by a previous use of %run. So we
201 201 also check explicitly that it really is a module:
202 202 """
203 203 _ip = get_ipython()
204 204 self.run_tmpfile()
205 205 nt.assert_equal(type(_ip.user_ns['__builtins__']),type(sys))
206 206
207 207 def test_run_profile( self ):
208 208 """Test that the option -p, which invokes the profiler, do not
209 209 crash by invoking execfile"""
210 210 self.run_tmpfile_p()
211 211
212 212 def test_run_debug_twice(self):
213 213 # https://github.com/ipython/ipython/issues/10028
214 214 _ip = get_ipython()
215 215 with tt.fake_input(['c']):
216 216 _ip.magic('run -d %s' % self.fname)
217 217 with tt.fake_input(['c']):
218 218 _ip.magic('run -d %s' % self.fname)
219 219
220 220
221 221 class TestMagicRunSimple(tt.TempFileMixin):
222 222
223 223 def test_simpledef(self):
224 224 """Test that simple class definitions work."""
225 225 src = ("class foo: pass\n"
226 226 "def f(): return foo()")
227 227 self.mktmp(src)
228 228 _ip.magic('run %s' % self.fname)
229 229 _ip.run_cell('t = isinstance(f(), foo)')
230 230 nt.assert_true(_ip.user_ns['t'])
231 231
232 232 def test_obj_del(self):
233 233 """Test that object's __del__ methods are called on exit."""
234 234 if sys.platform == 'win32':
235 235 try:
236 236 import win32api
237 237 except ImportError:
238 238 raise SkipTest("Test requires pywin32")
239 239 src = ("class A(object):\n"
240 240 " def __del__(self):\n"
241 241 " print 'object A deleted'\n"
242 242 "a = A()\n")
243 243 self.mktmp(py3compat.doctest_refactor_print(src))
244 244 if dec.module_not_available('sqlite3'):
245 245 err = 'WARNING: IPython History requires SQLite, your history will not be saved\n'
246 246 else:
247 247 err = None
248 248 tt.ipexec_validate(self.fname, 'object A deleted', err)
249 249
250 250 def test_aggressive_namespace_cleanup(self):
251 251 """Test that namespace cleanup is not too aggressive GH-238
252 252
253 253 Returning from another run magic deletes the namespace"""
254 254 # see ticket https://github.com/ipython/ipython/issues/238
255 255
256 256 with tt.TempFileMixin() as empty:
257 257 empty.mktmp('')
258 258 # On Windows, the filename will have \users in it, so we need to use the
259 259 # repr so that the \u becomes \\u.
260 260 src = ("ip = get_ipython()\n"
261 261 "for i in range(5):\n"
262 262 " try:\n"
263 263 " ip.magic(%r)\n"
264 264 " except NameError as e:\n"
265 265 " print(i)\n"
266 266 " break\n" % ('run ' + empty.fname))
267 267 self.mktmp(src)
268 268 _ip.magic('run %s' % self.fname)
269 269 _ip.run_cell('ip == get_ipython()')
270 270 nt.assert_equal(_ip.user_ns['i'], 4)
271 271
272 272 def test_run_second(self):
273 273 """Test that running a second file doesn't clobber the first, gh-3547
274 274 """
275 275 self.mktmp("avar = 1\n"
276 276 "def afunc():\n"
277 277 " return avar\n")
278 278
279 279 with tt.TempFileMixin() as empty:
280 280 empty.mktmp("")
281 281
282 282 _ip.magic('run %s' % self.fname)
283 283 _ip.magic('run %s' % empty.fname)
284 284 nt.assert_equal(_ip.user_ns['afunc'](), 1)
285 285
286 286 @dec.skip_win32
287 287 def test_tclass(self):
288 288 mydir = os.path.dirname(__file__)
289 289 tc = os.path.join(mydir, 'tclass')
290 290 src = ("%%run '%s' C-first\n"
291 291 "%%run '%s' C-second\n"
292 292 "%%run '%s' C-third\n") % (tc, tc, tc)
293 293 self.mktmp(src, '.ipy')
294 294 out = """\
295 295 ARGV 1-: ['C-first']
296 296 ARGV 1-: ['C-second']
297 297 tclass.py: deleting object: C-first
298 298 ARGV 1-: ['C-third']
299 299 tclass.py: deleting object: C-second
300 300 tclass.py: deleting object: C-third
301 301 """
302 302 if dec.module_not_available('sqlite3'):
303 303 err = 'WARNING: IPython History requires SQLite, your history will not be saved\n'
304 304 else:
305 305 err = None
306 306 tt.ipexec_validate(self.fname, out, err)
307 307
308 308 def test_run_i_after_reset(self):
309 309 """Check that %run -i still works after %reset (gh-693)"""
310 310 src = "yy = zz\n"
311 311 self.mktmp(src)
312 312 _ip.run_cell("zz = 23")
313 313 _ip.magic('run -i %s' % self.fname)
314 314 nt.assert_equal(_ip.user_ns['yy'], 23)
315 315 _ip.magic('reset -f')
316 316 _ip.run_cell("zz = 23")
317 317 _ip.magic('run -i %s' % self.fname)
318 318 nt.assert_equal(_ip.user_ns['yy'], 23)
319 319
320 320 def test_unicode(self):
321 321 """Check that files in odd encodings are accepted."""
322 322 mydir = os.path.dirname(__file__)
323 323 na = os.path.join(mydir, 'nonascii.py')
324 324 _ip.magic('run "%s"' % na)
325 325 nt.assert_equal(_ip.user_ns['u'], u'ΠŽΡ‚β„–Π€')
326 326
327 327 def test_run_py_file_attribute(self):
328 328 """Test handling of `__file__` attribute in `%run <file>.py`."""
329 329 src = "t = __file__\n"
330 330 self.mktmp(src)
331 331 _missing = object()
332 332 file1 = _ip.user_ns.get('__file__', _missing)
333 333 _ip.magic('run %s' % self.fname)
334 334 file2 = _ip.user_ns.get('__file__', _missing)
335 335
336 336 # Check that __file__ was equal to the filename in the script's
337 337 # namespace.
338 338 nt.assert_equal(_ip.user_ns['t'], self.fname)
339 339
340 340 # Check that __file__ was not leaked back into user_ns.
341 341 nt.assert_equal(file1, file2)
342 342
343 343 def test_run_ipy_file_attribute(self):
344 344 """Test handling of `__file__` attribute in `%run <file.ipy>`."""
345 345 src = "t = __file__\n"
346 346 self.mktmp(src, ext='.ipy')
347 347 _missing = object()
348 348 file1 = _ip.user_ns.get('__file__', _missing)
349 349 _ip.magic('run %s' % self.fname)
350 350 file2 = _ip.user_ns.get('__file__', _missing)
351 351
352 352 # Check that __file__ was equal to the filename in the script's
353 353 # namespace.
354 354 nt.assert_equal(_ip.user_ns['t'], self.fname)
355 355
356 356 # Check that __file__ was not leaked back into user_ns.
357 357 nt.assert_equal(file1, file2)
358 358
359 359 def test_run_formatting(self):
360 360 """ Test that %run -t -N<N> does not raise a TypeError for N > 1."""
361 361 src = "pass"
362 362 self.mktmp(src)
363 363 _ip.magic('run -t -N 1 %s' % self.fname)
364 364 _ip.magic('run -t -N 10 %s' % self.fname)
365 365
366 366 def test_ignore_sys_exit(self):
367 367 """Test the -e option to ignore sys.exit()"""
368 368 src = "import sys; sys.exit(1)"
369 369 self.mktmp(src)
370 370 with tt.AssertPrints('SystemExit'):
371 371 _ip.magic('run %s' % self.fname)
372 372
373 373 with tt.AssertNotPrints('SystemExit'):
374 374 _ip.magic('run -e %s' % self.fname)
375 375
376 376 def test_run_nb(self):
377 377 """Test %run notebook.ipynb"""
378 378 from nbformat import v4, writes
379 379 nb = v4.new_notebook(
380 380 cells=[
381 381 v4.new_markdown_cell("The Ultimate Question of Everything"),
382 382 v4.new_code_cell("answer=42")
383 383 ]
384 384 )
385 385 src = writes(nb, version=4)
386 386 self.mktmp(src, ext='.ipynb')
387 387
388 388 _ip.magic("run %s" % self.fname)
389 389
390 390 nt.assert_equal(_ip.user_ns['answer'], 42)
391 391
392 392
393 393
394 394 class TestMagicRunWithPackage(unittest.TestCase):
395 395
396 396 def writefile(self, name, content):
397 397 path = os.path.join(self.tempdir.name, name)
398 398 d = os.path.dirname(path)
399 399 if not os.path.isdir(d):
400 400 os.makedirs(d)
401 401 with open(path, 'w') as f:
402 402 f.write(textwrap.dedent(content))
403 403
404 404 def setUp(self):
405 405 self.package = package = 'tmp{0}'.format(repr(random.random())[2:])
406 406 """Temporary valid python package name."""
407 407
408 408 self.value = int(random.random() * 10000)
409 409
410 410 self.tempdir = TemporaryDirectory()
411 self.__orig_cwd = py3compat.getcwd()
411 self.__orig_cwd = os.getcwd()
412 412 sys.path.insert(0, self.tempdir.name)
413 413
414 414 self.writefile(os.path.join(package, '__init__.py'), '')
415 415 self.writefile(os.path.join(package, 'sub.py'), """
416 416 x = {0!r}
417 417 """.format(self.value))
418 418 self.writefile(os.path.join(package, 'relative.py'), """
419 419 from .sub import x
420 420 """)
421 421 self.writefile(os.path.join(package, 'absolute.py'), """
422 422 from {0}.sub import x
423 423 """.format(package))
424 424
425 425 def tearDown(self):
426 426 os.chdir(self.__orig_cwd)
427 427 sys.path[:] = [p for p in sys.path if p != self.tempdir.name]
428 428 self.tempdir.cleanup()
429 429
430 430 def check_run_submodule(self, submodule, opts=''):
431 431 _ip.user_ns.pop('x', None)
432 432 _ip.magic('run {2} -m {0}.{1}'.format(self.package, submodule, opts))
433 433 self.assertEqual(_ip.user_ns['x'], self.value,
434 434 'Variable `x` is not loaded from module `{0}`.'
435 435 .format(submodule))
436 436
437 437 def test_run_submodule_with_absolute_import(self):
438 438 self.check_run_submodule('absolute')
439 439
440 440 def test_run_submodule_with_relative_import(self):
441 441 """Run submodule that has a relative import statement (#2727)."""
442 442 self.check_run_submodule('relative')
443 443
444 444 def test_prun_submodule_with_absolute_import(self):
445 445 self.check_run_submodule('absolute', '-p')
446 446
447 447 def test_prun_submodule_with_relative_import(self):
448 448 self.check_run_submodule('relative', '-p')
449 449
450 450 def with_fake_debugger(func):
451 451 @functools.wraps(func)
452 452 def wrapper(*args, **kwds):
453 453 with patch.object(debugger.Pdb, 'run', staticmethod(eval)):
454 454 return func(*args, **kwds)
455 455 return wrapper
456 456
457 457 @with_fake_debugger
458 458 def test_debug_run_submodule_with_absolute_import(self):
459 459 self.check_run_submodule('absolute', '-d')
460 460
461 461 @with_fake_debugger
462 462 def test_debug_run_submodule_with_relative_import(self):
463 463 self.check_run_submodule('relative', '-d')
464 464
465 465 def test_run__name__():
466 466 with TemporaryDirectory() as td:
467 467 path = pjoin(td, 'foo.py')
468 468 with open(path, 'w') as f:
469 469 f.write("q = __name__")
470 470
471 471 _ip.user_ns.pop('q', None)
472 472 _ip.magic('run {}'.format(path))
473 473 nt.assert_equal(_ip.user_ns.pop('q'), '__main__')
474 474
475 475 _ip.magic('run -n {}'.format(path))
476 476 nt.assert_equal(_ip.user_ns.pop('q'), 'foo')
477 477
478 478 def test_run_tb():
479 479 """Test traceback offset in %run"""
480 480 with TemporaryDirectory() as td:
481 481 path = pjoin(td, 'foo.py')
482 482 with open(path, 'w') as f:
483 483 f.write('\n'.join([
484 484 "def foo():",
485 485 " return bar()",
486 486 "def bar():",
487 487 " raise RuntimeError('hello!')",
488 488 "foo()",
489 489 ]))
490 490 with capture_output() as io:
491 491 _ip.magic('run {}'.format(path))
492 492 out = io.stdout
493 493 nt.assert_not_in("execfile", out)
494 494 nt.assert_in("RuntimeError", out)
495 495 nt.assert_equal(out.count("---->"), 3)
496 496
497 497 @dec.knownfailureif(sys.platform == 'win32', "writes to io.stdout aren't captured on Windows")
498 498 def test_script_tb():
499 499 """Test traceback offset in `ipython script.py`"""
500 500 with TemporaryDirectory() as td:
501 501 path = pjoin(td, 'foo.py')
502 502 with open(path, 'w') as f:
503 503 f.write('\n'.join([
504 504 "def foo():",
505 505 " return bar()",
506 506 "def bar():",
507 507 " raise RuntimeError('hello!')",
508 508 "foo()",
509 509 ]))
510 510 out, err = tt.ipexec(path)
511 511 nt.assert_not_in("execfile", out)
512 512 nt.assert_in("RuntimeError", out)
513 513 nt.assert_equal(out.count("---->"), 3)
514 514
@@ -1,191 +1,191 b''
1 1 """Windows-specific implementation of process utilities.
2 2
3 3 This file is only meant to be imported by process.py, not by end-users.
4 4 """
5 5
6 6 #-----------------------------------------------------------------------------
7 7 # Copyright (C) 2010-2011 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Imports
15 15 #-----------------------------------------------------------------------------
16 16
17 17 # stdlib
18 18 import os
19 19 import sys
20 20 import ctypes
21 21
22 22 from ctypes import c_int, POINTER
23 23 from ctypes.wintypes import LPCWSTR, HLOCAL
24 24 from subprocess import STDOUT
25 25
26 26 # our own imports
27 27 from ._process_common import read_no_interrupt, process_handler, arg_split as py_arg_split
28 28 from . import py3compat
29 29 from .encoding import DEFAULT_ENCODING
30 30
31 31 #-----------------------------------------------------------------------------
32 32 # Function definitions
33 33 #-----------------------------------------------------------------------------
34 34
35 35 class AvoidUNCPath(object):
36 36 """A context manager to protect command execution from UNC paths.
37 37
38 38 In the Win32 API, commands can't be invoked with the cwd being a UNC path.
39 39 This context manager temporarily changes directory to the 'C:' drive on
40 40 entering, and restores the original working directory on exit.
41 41
42 42 The context manager returns the starting working directory *if* it made a
43 43 change and None otherwise, so that users can apply the necessary adjustment
44 44 to their system calls in the event of a change.
45 45
46 46 Examples
47 47 --------
48 48 ::
49 49 cmd = 'dir'
50 50 with AvoidUNCPath() as path:
51 51 if path is not None:
52 52 cmd = '"pushd %s &&"%s' % (path, cmd)
53 53 os.system(cmd)
54 54 """
55 55 def __enter__(self):
56 self.path = py3compat.getcwd()
56 self.path = os.getcwd()
57 57 self.is_unc_path = self.path.startswith(r"\\")
58 58 if self.is_unc_path:
59 59 # change to c drive (as cmd.exe cannot handle UNC addresses)
60 60 os.chdir("C:")
61 61 return self.path
62 62 else:
63 63 # We return None to signal that there was no change in the working
64 64 # directory
65 65 return None
66 66
67 67 def __exit__(self, exc_type, exc_value, traceback):
68 68 if self.is_unc_path:
69 69 os.chdir(self.path)
70 70
71 71
72 72 def _find_cmd(cmd):
73 73 """Find the full path to a .bat or .exe using the win32api module."""
74 74 try:
75 75 from win32api import SearchPath
76 76 except ImportError:
77 77 raise ImportError('you need to have pywin32 installed for this to work')
78 78 else:
79 79 PATH = os.environ['PATH']
80 80 extensions = ['.exe', '.com', '.bat', '.py']
81 81 path = None
82 82 for ext in extensions:
83 83 try:
84 84 path = SearchPath(PATH, cmd, ext)[0]
85 85 except:
86 86 pass
87 87 if path is None:
88 88 raise OSError("command %r not found" % cmd)
89 89 else:
90 90 return path
91 91
92 92
93 93 def _system_body(p):
94 94 """Callback for _system."""
95 95 enc = DEFAULT_ENCODING
96 96 for line in read_no_interrupt(p.stdout).splitlines():
97 97 line = line.decode(enc, 'replace')
98 98 print(line, file=sys.stdout)
99 99 for line in read_no_interrupt(p.stderr).splitlines():
100 100 line = line.decode(enc, 'replace')
101 101 print(line, file=sys.stderr)
102 102
103 103 # Wait to finish for returncode
104 104 return p.wait()
105 105
106 106
107 107 def system(cmd):
108 108 """Win32 version of os.system() that works with network shares.
109 109
110 110 Note that this implementation returns None, as meant for use in IPython.
111 111
112 112 Parameters
113 113 ----------
114 114 cmd : str or list
115 115 A command to be executed in the system shell.
116 116
117 117 Returns
118 118 -------
119 119 None : we explicitly do NOT return the subprocess status code, as this
120 120 utility is meant to be used extensively in IPython, where any return value
121 121 would trigger :func:`sys.displayhook` calls.
122 122 """
123 123 # The controller provides interactivity with both
124 124 # stdin and stdout
125 125 #import _process_win32_controller
126 126 #_process_win32_controller.system(cmd)
127 127
128 128 with AvoidUNCPath() as path:
129 129 if path is not None:
130 130 cmd = '"pushd %s &&"%s' % (path, cmd)
131 131 return process_handler(cmd, _system_body)
132 132
133 133 def getoutput(cmd):
134 134 """Return standard output of executing cmd in a shell.
135 135
136 136 Accepts the same arguments as os.system().
137 137
138 138 Parameters
139 139 ----------
140 140 cmd : str or list
141 141 A command to be executed in the system shell.
142 142
143 143 Returns
144 144 -------
145 145 stdout : str
146 146 """
147 147
148 148 with AvoidUNCPath() as path:
149 149 if path is not None:
150 150 cmd = '"pushd %s &&"%s' % (path, cmd)
151 151 out = process_handler(cmd, lambda p: p.communicate()[0], STDOUT)
152 152
153 153 if out is None:
154 154 out = b''
155 155 return py3compat.bytes_to_str(out)
156 156
157 157 try:
158 158 CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
159 159 CommandLineToArgvW.arg_types = [LPCWSTR, POINTER(c_int)]
160 160 CommandLineToArgvW.restype = POINTER(LPCWSTR)
161 161 LocalFree = ctypes.windll.kernel32.LocalFree
162 162 LocalFree.res_type = HLOCAL
163 163 LocalFree.arg_types = [HLOCAL]
164 164
165 165 def arg_split(commandline, posix=False, strict=True):
166 166 """Split a command line's arguments in a shell-like manner.
167 167
168 168 This is a special version for windows that use a ctypes call to CommandLineToArgvW
169 169 to do the argv splitting. The posix paramter is ignored.
170 170
171 171 If strict=False, process_common.arg_split(...strict=False) is used instead.
172 172 """
173 173 #CommandLineToArgvW returns path to executable if called with empty string.
174 174 if commandline.strip() == "":
175 175 return []
176 176 if not strict:
177 177 # not really a cl-arg, fallback on _process_common
178 178 return py_arg_split(commandline, posix=posix, strict=strict)
179 179 argvn = c_int()
180 180 result_pointer = CommandLineToArgvW(py3compat.cast_unicode(commandline.lstrip()), ctypes.byref(argvn))
181 181 result_array_type = LPCWSTR * argvn.value
182 182 result = [arg for arg in result_array_type.from_address(ctypes.addressof(result_pointer.contents))]
183 183 retval = LocalFree(result_pointer)
184 184 return result
185 185 except AttributeError:
186 186 arg_split = py_arg_split
187 187
188 188 def check_pid(pid):
189 189 # OpenProcess returns 0 if no such process (of ours) exists
190 190 # positive int otherwise
191 191 return bool(ctypes.windll.kernel32.OpenProcess(1,0,pid))
@@ -1,576 +1,576 b''
1 1 """Windows-specific implementation of process utilities with direct WinAPI.
2 2
3 3 This file is meant to be used by process.py
4 4 """
5 5
6 6 #-----------------------------------------------------------------------------
7 7 # Copyright (C) 2010-2011 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13
14 14 # stdlib
15 15 import os, sys, threading
16 16 import ctypes, msvcrt
17 17
18 18 # local imports
19 19 from . import py3compat
20 20
21 21 # Win32 API types needed for the API calls
22 22 from ctypes import POINTER
23 23 from ctypes.wintypes import HANDLE, HLOCAL, LPVOID, WORD, DWORD, BOOL, \
24 24 ULONG, LPCWSTR
25 25 LPDWORD = POINTER(DWORD)
26 26 LPHANDLE = POINTER(HANDLE)
27 27 ULONG_PTR = POINTER(ULONG)
28 28 class SECURITY_ATTRIBUTES(ctypes.Structure):
29 29 _fields_ = [("nLength", DWORD),
30 30 ("lpSecurityDescriptor", LPVOID),
31 31 ("bInheritHandle", BOOL)]
32 32 LPSECURITY_ATTRIBUTES = POINTER(SECURITY_ATTRIBUTES)
33 33 class STARTUPINFO(ctypes.Structure):
34 34 _fields_ = [("cb", DWORD),
35 35 ("lpReserved", LPCWSTR),
36 36 ("lpDesktop", LPCWSTR),
37 37 ("lpTitle", LPCWSTR),
38 38 ("dwX", DWORD),
39 39 ("dwY", DWORD),
40 40 ("dwXSize", DWORD),
41 41 ("dwYSize", DWORD),
42 42 ("dwXCountChars", DWORD),
43 43 ("dwYCountChars", DWORD),
44 44 ("dwFillAttribute", DWORD),
45 45 ("dwFlags", DWORD),
46 46 ("wShowWindow", WORD),
47 47 ("cbReserved2", WORD),
48 48 ("lpReserved2", LPVOID),
49 49 ("hStdInput", HANDLE),
50 50 ("hStdOutput", HANDLE),
51 51 ("hStdError", HANDLE)]
52 52 LPSTARTUPINFO = POINTER(STARTUPINFO)
53 53 class PROCESS_INFORMATION(ctypes.Structure):
54 54 _fields_ = [("hProcess", HANDLE),
55 55 ("hThread", HANDLE),
56 56 ("dwProcessId", DWORD),
57 57 ("dwThreadId", DWORD)]
58 58 LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION)
59 59
60 60 # Win32 API constants needed
61 61 ERROR_HANDLE_EOF = 38
62 62 ERROR_BROKEN_PIPE = 109
63 63 ERROR_NO_DATA = 232
64 64 HANDLE_FLAG_INHERIT = 0x0001
65 65 STARTF_USESTDHANDLES = 0x0100
66 66 CREATE_SUSPENDED = 0x0004
67 67 CREATE_NEW_CONSOLE = 0x0010
68 68 CREATE_NO_WINDOW = 0x08000000
69 69 STILL_ACTIVE = 259
70 70 WAIT_TIMEOUT = 0x0102
71 71 WAIT_FAILED = 0xFFFFFFFF
72 72 INFINITE = 0xFFFFFFFF
73 73 DUPLICATE_SAME_ACCESS = 0x00000002
74 74 ENABLE_ECHO_INPUT = 0x0004
75 75 ENABLE_LINE_INPUT = 0x0002
76 76 ENABLE_PROCESSED_INPUT = 0x0001
77 77
78 78 # Win32 API functions needed
79 79 GetLastError = ctypes.windll.kernel32.GetLastError
80 80 GetLastError.argtypes = []
81 81 GetLastError.restype = DWORD
82 82
83 83 CreateFile = ctypes.windll.kernel32.CreateFileW
84 84 CreateFile.argtypes = [LPCWSTR, DWORD, DWORD, LPVOID, DWORD, DWORD, HANDLE]
85 85 CreateFile.restype = HANDLE
86 86
87 87 CreatePipe = ctypes.windll.kernel32.CreatePipe
88 88 CreatePipe.argtypes = [POINTER(HANDLE), POINTER(HANDLE),
89 89 LPSECURITY_ATTRIBUTES, DWORD]
90 90 CreatePipe.restype = BOOL
91 91
92 92 CreateProcess = ctypes.windll.kernel32.CreateProcessW
93 93 CreateProcess.argtypes = [LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES,
94 94 LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCWSTR, LPSTARTUPINFO,
95 95 LPPROCESS_INFORMATION]
96 96 CreateProcess.restype = BOOL
97 97
98 98 GetExitCodeProcess = ctypes.windll.kernel32.GetExitCodeProcess
99 99 GetExitCodeProcess.argtypes = [HANDLE, LPDWORD]
100 100 GetExitCodeProcess.restype = BOOL
101 101
102 102 GetCurrentProcess = ctypes.windll.kernel32.GetCurrentProcess
103 103 GetCurrentProcess.argtypes = []
104 104 GetCurrentProcess.restype = HANDLE
105 105
106 106 ResumeThread = ctypes.windll.kernel32.ResumeThread
107 107 ResumeThread.argtypes = [HANDLE]
108 108 ResumeThread.restype = DWORD
109 109
110 110 ReadFile = ctypes.windll.kernel32.ReadFile
111 111 ReadFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPVOID]
112 112 ReadFile.restype = BOOL
113 113
114 114 WriteFile = ctypes.windll.kernel32.WriteFile
115 115 WriteFile.argtypes = [HANDLE, LPVOID, DWORD, LPDWORD, LPVOID]
116 116 WriteFile.restype = BOOL
117 117
118 118 GetConsoleMode = ctypes.windll.kernel32.GetConsoleMode
119 119 GetConsoleMode.argtypes = [HANDLE, LPDWORD]
120 120 GetConsoleMode.restype = BOOL
121 121
122 122 SetConsoleMode = ctypes.windll.kernel32.SetConsoleMode
123 123 SetConsoleMode.argtypes = [HANDLE, DWORD]
124 124 SetConsoleMode.restype = BOOL
125 125
126 126 FlushConsoleInputBuffer = ctypes.windll.kernel32.FlushConsoleInputBuffer
127 127 FlushConsoleInputBuffer.argtypes = [HANDLE]
128 128 FlushConsoleInputBuffer.restype = BOOL
129 129
130 130 WaitForSingleObject = ctypes.windll.kernel32.WaitForSingleObject
131 131 WaitForSingleObject.argtypes = [HANDLE, DWORD]
132 132 WaitForSingleObject.restype = DWORD
133 133
134 134 DuplicateHandle = ctypes.windll.kernel32.DuplicateHandle
135 135 DuplicateHandle.argtypes = [HANDLE, HANDLE, HANDLE, LPHANDLE,
136 136 DWORD, BOOL, DWORD]
137 137 DuplicateHandle.restype = BOOL
138 138
139 139 SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation
140 140 SetHandleInformation.argtypes = [HANDLE, DWORD, DWORD]
141 141 SetHandleInformation.restype = BOOL
142 142
143 143 CloseHandle = ctypes.windll.kernel32.CloseHandle
144 144 CloseHandle.argtypes = [HANDLE]
145 145 CloseHandle.restype = BOOL
146 146
147 147 CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
148 148 CommandLineToArgvW.argtypes = [LPCWSTR, POINTER(ctypes.c_int)]
149 149 CommandLineToArgvW.restype = POINTER(LPCWSTR)
150 150
151 151 LocalFree = ctypes.windll.kernel32.LocalFree
152 152 LocalFree.argtypes = [HLOCAL]
153 153 LocalFree.restype = HLOCAL
154 154
155 155 class AvoidUNCPath(object):
156 156 """A context manager to protect command execution from UNC paths.
157 157
158 158 In the Win32 API, commands can't be invoked with the cwd being a UNC path.
159 159 This context manager temporarily changes directory to the 'C:' drive on
160 160 entering, and restores the original working directory on exit.
161 161
162 162 The context manager returns the starting working directory *if* it made a
163 163 change and None otherwise, so that users can apply the necessary adjustment
164 164 to their system calls in the event of a change.
165 165
166 166 Examples
167 167 --------
168 168 ::
169 169 cmd = 'dir'
170 170 with AvoidUNCPath() as path:
171 171 if path is not None:
172 172 cmd = '"pushd %s &&"%s' % (path, cmd)
173 173 os.system(cmd)
174 174 """
175 175 def __enter__(self):
176 self.path = py3compat.getcwd()
176 self.path = os.getcwd()
177 177 self.is_unc_path = self.path.startswith(r"\\")
178 178 if self.is_unc_path:
179 179 # change to c drive (as cmd.exe cannot handle UNC addresses)
180 180 os.chdir("C:")
181 181 return self.path
182 182 else:
183 183 # We return None to signal that there was no change in the working
184 184 # directory
185 185 return None
186 186
187 187 def __exit__(self, exc_type, exc_value, traceback):
188 188 if self.is_unc_path:
189 189 os.chdir(self.path)
190 190
191 191
192 192 class Win32ShellCommandController(object):
193 193 """Runs a shell command in a 'with' context.
194 194
195 195 This implementation is Win32-specific.
196 196
197 197 Example:
198 198 # Runs the command interactively with default console stdin/stdout
199 199 with ShellCommandController('python -i') as scc:
200 200 scc.run()
201 201
202 202 # Runs the command using the provided functions for stdin/stdout
203 203 def my_stdout_func(s):
204 204 # print or save the string 's'
205 205 write_to_stdout(s)
206 206 def my_stdin_func():
207 207 # If input is available, return it as a string.
208 208 if input_available():
209 209 return get_input()
210 210 # If no input available, return None after a short delay to
211 211 # keep from blocking.
212 212 else:
213 213 time.sleep(0.01)
214 214 return None
215 215
216 216 with ShellCommandController('python -i') as scc:
217 217 scc.run(my_stdout_func, my_stdin_func)
218 218 """
219 219
220 220 def __init__(self, cmd, mergeout = True):
221 221 """Initializes the shell command controller.
222 222
223 223 The cmd is the program to execute, and mergeout is
224 224 whether to blend stdout and stderr into one output
225 225 in stdout. Merging them together in this fashion more
226 226 reliably keeps stdout and stderr in the correct order
227 227 especially for interactive shell usage.
228 228 """
229 229 self.cmd = cmd
230 230 self.mergeout = mergeout
231 231
232 232 def __enter__(self):
233 233 cmd = self.cmd
234 234 mergeout = self.mergeout
235 235
236 236 self.hstdout, self.hstdin, self.hstderr = None, None, None
237 237 self.piProcInfo = None
238 238 try:
239 239 p_hstdout, c_hstdout, p_hstderr, \
240 240 c_hstderr, p_hstdin, c_hstdin = [None]*6
241 241
242 242 # SECURITY_ATTRIBUTES with inherit handle set to True
243 243 saAttr = SECURITY_ATTRIBUTES()
244 244 saAttr.nLength = ctypes.sizeof(saAttr)
245 245 saAttr.bInheritHandle = True
246 246 saAttr.lpSecurityDescriptor = None
247 247
248 248 def create_pipe(uninherit):
249 249 """Creates a Windows pipe, which consists of two handles.
250 250
251 251 The 'uninherit' parameter controls which handle is not
252 252 inherited by the child process.
253 253 """
254 254 handles = HANDLE(), HANDLE()
255 255 if not CreatePipe(ctypes.byref(handles[0]),
256 256 ctypes.byref(handles[1]), ctypes.byref(saAttr), 0):
257 257 raise ctypes.WinError()
258 258 if not SetHandleInformation(handles[uninherit],
259 259 HANDLE_FLAG_INHERIT, 0):
260 260 raise ctypes.WinError()
261 261 return handles[0].value, handles[1].value
262 262
263 263 p_hstdout, c_hstdout = create_pipe(uninherit=0)
264 264 # 'mergeout' signals that stdout and stderr should be merged.
265 265 # We do that by using one pipe for both of them.
266 266 if mergeout:
267 267 c_hstderr = HANDLE()
268 268 if not DuplicateHandle(GetCurrentProcess(), c_hstdout,
269 269 GetCurrentProcess(), ctypes.byref(c_hstderr),
270 270 0, True, DUPLICATE_SAME_ACCESS):
271 271 raise ctypes.WinError()
272 272 else:
273 273 p_hstderr, c_hstderr = create_pipe(uninherit=0)
274 274 c_hstdin, p_hstdin = create_pipe(uninherit=1)
275 275
276 276 # Create the process object
277 277 piProcInfo = PROCESS_INFORMATION()
278 278 siStartInfo = STARTUPINFO()
279 279 siStartInfo.cb = ctypes.sizeof(siStartInfo)
280 280 siStartInfo.hStdInput = c_hstdin
281 281 siStartInfo.hStdOutput = c_hstdout
282 282 siStartInfo.hStdError = c_hstderr
283 283 siStartInfo.dwFlags = STARTF_USESTDHANDLES
284 284 dwCreationFlags = CREATE_SUSPENDED | CREATE_NO_WINDOW # | CREATE_NEW_CONSOLE
285 285
286 286 if not CreateProcess(None,
287 287 u"cmd.exe /c " + cmd,
288 288 None, None, True, dwCreationFlags,
289 289 None, None, ctypes.byref(siStartInfo),
290 290 ctypes.byref(piProcInfo)):
291 291 raise ctypes.WinError()
292 292
293 293 # Close this process's versions of the child handles
294 294 CloseHandle(c_hstdin)
295 295 c_hstdin = None
296 296 CloseHandle(c_hstdout)
297 297 c_hstdout = None
298 298 if c_hstderr is not None:
299 299 CloseHandle(c_hstderr)
300 300 c_hstderr = None
301 301
302 302 # Transfer ownership of the parent handles to the object
303 303 self.hstdin = p_hstdin
304 304 p_hstdin = None
305 305 self.hstdout = p_hstdout
306 306 p_hstdout = None
307 307 if not mergeout:
308 308 self.hstderr = p_hstderr
309 309 p_hstderr = None
310 310 self.piProcInfo = piProcInfo
311 311
312 312 finally:
313 313 if p_hstdin:
314 314 CloseHandle(p_hstdin)
315 315 if c_hstdin:
316 316 CloseHandle(c_hstdin)
317 317 if p_hstdout:
318 318 CloseHandle(p_hstdout)
319 319 if c_hstdout:
320 320 CloseHandle(c_hstdout)
321 321 if p_hstderr:
322 322 CloseHandle(p_hstderr)
323 323 if c_hstderr:
324 324 CloseHandle(c_hstderr)
325 325
326 326 return self
327 327
328 328 def _stdin_thread(self, handle, hprocess, func, stdout_func):
329 329 exitCode = DWORD()
330 330 bytesWritten = DWORD(0)
331 331 while True:
332 332 #print("stdin thread loop start")
333 333 # Get the input string (may be bytes or unicode)
334 334 data = func()
335 335
336 336 # None signals to poll whether the process has exited
337 337 if data is None:
338 338 #print("checking for process completion")
339 339 if not GetExitCodeProcess(hprocess, ctypes.byref(exitCode)):
340 340 raise ctypes.WinError()
341 341 if exitCode.value != STILL_ACTIVE:
342 342 return
343 343 # TESTING: Does zero-sized writefile help?
344 344 if not WriteFile(handle, "", 0,
345 345 ctypes.byref(bytesWritten), None):
346 346 raise ctypes.WinError()
347 347 continue
348 348 #print("\nGot str %s\n" % repr(data), file=sys.stderr)
349 349
350 350 # Encode the string to the console encoding
351 351 if isinstance(data, unicode): #FIXME: Python3
352 352 data = data.encode('utf_8')
353 353
354 354 # What we have now must be a string of bytes
355 355 if not isinstance(data, str): #FIXME: Python3
356 356 raise RuntimeError("internal stdin function string error")
357 357
358 358 # An empty string signals EOF
359 359 if len(data) == 0:
360 360 return
361 361
362 362 # In a windows console, sometimes the input is echoed,
363 363 # but sometimes not. How do we determine when to do this?
364 364 stdout_func(data)
365 365 # WriteFile may not accept all the data at once.
366 366 # Loop until everything is processed
367 367 while len(data) != 0:
368 368 #print("Calling writefile")
369 369 if not WriteFile(handle, data, len(data),
370 370 ctypes.byref(bytesWritten), None):
371 371 # This occurs at exit
372 372 if GetLastError() == ERROR_NO_DATA:
373 373 return
374 374 raise ctypes.WinError()
375 375 #print("Called writefile")
376 376 data = data[bytesWritten.value:]
377 377
378 378 def _stdout_thread(self, handle, func):
379 379 # Allocate the output buffer
380 380 data = ctypes.create_string_buffer(4096)
381 381 while True:
382 382 bytesRead = DWORD(0)
383 383 if not ReadFile(handle, data, 4096,
384 384 ctypes.byref(bytesRead), None):
385 385 le = GetLastError()
386 386 if le == ERROR_BROKEN_PIPE:
387 387 return
388 388 else:
389 389 raise ctypes.WinError()
390 390 # FIXME: Python3
391 391 s = data.value[0:bytesRead.value]
392 392 #print("\nv: %s" % repr(s), file=sys.stderr)
393 393 func(s.decode('utf_8', 'replace'))
394 394
395 395 def run(self, stdout_func = None, stdin_func = None, stderr_func = None):
396 396 """Runs the process, using the provided functions for I/O.
397 397
398 398 The function stdin_func should return strings whenever a
399 399 character or characters become available.
400 400 The functions stdout_func and stderr_func are called whenever
401 401 something is printed to stdout or stderr, respectively.
402 402 These functions are called from different threads (but not
403 403 concurrently, because of the GIL).
404 404 """
405 405 if stdout_func is None and stdin_func is None and stderr_func is None:
406 406 return self._run_stdio()
407 407
408 408 if stderr_func is not None and self.mergeout:
409 409 raise RuntimeError("Shell command was initiated with "
410 410 "merged stdin/stdout, but a separate stderr_func "
411 411 "was provided to the run() method")
412 412
413 413 # Create a thread for each input/output handle
414 414 stdin_thread = None
415 415 threads = []
416 416 if stdin_func:
417 417 stdin_thread = threading.Thread(target=self._stdin_thread,
418 418 args=(self.hstdin, self.piProcInfo.hProcess,
419 419 stdin_func, stdout_func))
420 420 threads.append(threading.Thread(target=self._stdout_thread,
421 421 args=(self.hstdout, stdout_func)))
422 422 if not self.mergeout:
423 423 if stderr_func is None:
424 424 stderr_func = stdout_func
425 425 threads.append(threading.Thread(target=self._stdout_thread,
426 426 args=(self.hstderr, stderr_func)))
427 427 # Start the I/O threads and the process
428 428 if ResumeThread(self.piProcInfo.hThread) == 0xFFFFFFFF:
429 429 raise ctypes.WinError()
430 430 if stdin_thread is not None:
431 431 stdin_thread.start()
432 432 for thread in threads:
433 433 thread.start()
434 434 # Wait for the process to complete
435 435 if WaitForSingleObject(self.piProcInfo.hProcess, INFINITE) == \
436 436 WAIT_FAILED:
437 437 raise ctypes.WinError()
438 438 # Wait for the I/O threads to complete
439 439 for thread in threads:
440 440 thread.join()
441 441
442 442 # Wait for the stdin thread to complete
443 443 if stdin_thread is not None:
444 444 stdin_thread.join()
445 445
446 446 def _stdin_raw_nonblock(self):
447 447 """Use the raw Win32 handle of sys.stdin to do non-blocking reads"""
448 448 # WARNING: This is experimental, and produces inconsistent results.
449 449 # It's possible for the handle not to be appropriate for use
450 450 # with WaitForSingleObject, among other things.
451 451 handle = msvcrt.get_osfhandle(sys.stdin.fileno())
452 452 result = WaitForSingleObject(handle, 100)
453 453 if result == WAIT_FAILED:
454 454 raise ctypes.WinError()
455 455 elif result == WAIT_TIMEOUT:
456 456 print(".", end='')
457 457 return None
458 458 else:
459 459 data = ctypes.create_string_buffer(256)
460 460 bytesRead = DWORD(0)
461 461 print('?', end='')
462 462
463 463 if not ReadFile(handle, data, 256,
464 464 ctypes.byref(bytesRead), None):
465 465 raise ctypes.WinError()
466 466 # This ensures the non-blocking works with an actual console
467 467 # Not checking the error, so the processing will still work with
468 468 # other handle types
469 469 FlushConsoleInputBuffer(handle)
470 470
471 471 data = data.value
472 472 data = data.replace('\r\n', '\n')
473 473 data = data.replace('\r', '\n')
474 474 print(repr(data) + " ", end='')
475 475 return data
476 476
477 477 def _stdin_raw_block(self):
478 478 """Use a blocking stdin read"""
479 479 # The big problem with the blocking read is that it doesn't
480 480 # exit when it's supposed to in all contexts. An extra
481 481 # key-press may be required to trigger the exit.
482 482 try:
483 483 data = sys.stdin.read(1)
484 484 data = data.replace('\r', '\n')
485 485 return data
486 486 except WindowsError as we:
487 487 if we.winerror == ERROR_NO_DATA:
488 488 # This error occurs when the pipe is closed
489 489 return None
490 490 else:
491 491 # Otherwise let the error propagate
492 492 raise we
493 493
494 494 def _stdout_raw(self, s):
495 495 """Writes the string to stdout"""
496 496 print(s, end='', file=sys.stdout)
497 497 sys.stdout.flush()
498 498
499 499 def _stderr_raw(self, s):
500 500 """Writes the string to stdout"""
501 501 print(s, end='', file=sys.stderr)
502 502 sys.stderr.flush()
503 503
504 504 def _run_stdio(self):
505 505 """Runs the process using the system standard I/O.
506 506
507 507 IMPORTANT: stdin needs to be asynchronous, so the Python
508 508 sys.stdin object is not used. Instead,
509 509 msvcrt.kbhit/getwch are used asynchronously.
510 510 """
511 511 # Disable Line and Echo mode
512 512 #lpMode = DWORD()
513 513 #handle = msvcrt.get_osfhandle(sys.stdin.fileno())
514 514 #if GetConsoleMode(handle, ctypes.byref(lpMode)):
515 515 # set_console_mode = True
516 516 # if not SetConsoleMode(handle, lpMode.value &
517 517 # ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT)):
518 518 # raise ctypes.WinError()
519 519
520 520 if self.mergeout:
521 521 return self.run(stdout_func = self._stdout_raw,
522 522 stdin_func = self._stdin_raw_block)
523 523 else:
524 524 return self.run(stdout_func = self._stdout_raw,
525 525 stdin_func = self._stdin_raw_block,
526 526 stderr_func = self._stderr_raw)
527 527
528 528 # Restore the previous console mode
529 529 #if set_console_mode:
530 530 # if not SetConsoleMode(handle, lpMode.value):
531 531 # raise ctypes.WinError()
532 532
533 533 def __exit__(self, exc_type, exc_value, traceback):
534 534 if self.hstdin:
535 535 CloseHandle(self.hstdin)
536 536 self.hstdin = None
537 537 if self.hstdout:
538 538 CloseHandle(self.hstdout)
539 539 self.hstdout = None
540 540 if self.hstderr:
541 541 CloseHandle(self.hstderr)
542 542 self.hstderr = None
543 543 if self.piProcInfo is not None:
544 544 CloseHandle(self.piProcInfo.hProcess)
545 545 CloseHandle(self.piProcInfo.hThread)
546 546 self.piProcInfo = None
547 547
548 548
549 549 def system(cmd):
550 550 """Win32 version of os.system() that works with network shares.
551 551
552 552 Note that this implementation returns None, as meant for use in IPython.
553 553
554 554 Parameters
555 555 ----------
556 556 cmd : str
557 557 A command to be executed in the system shell.
558 558
559 559 Returns
560 560 -------
561 561 None : we explicitly do NOT return the subprocess status code, as this
562 562 utility is meant to be used extensively in IPython, where any return value
563 563 would trigger :func:`sys.displayhook` calls.
564 564 """
565 565 with AvoidUNCPath() as path:
566 566 if path is not None:
567 567 cmd = '"pushd %s &&"%s' % (path, cmd)
568 568 with Win32ShellCommandController(cmd) as scc:
569 569 scc.run()
570 570
571 571
572 572 if __name__ == "__main__":
573 573 print("Test starting!")
574 574 #system("cmd")
575 575 system("python -i")
576 576 print("Test finished!")
@@ -1,447 +1,447 b''
1 1 # encoding: utf-8
2 2 """
3 3 Utilities for path handling.
4 4 """
5 5
6 6 # Copyright (c) IPython Development Team.
7 7 # Distributed under the terms of the Modified BSD License.
8 8
9 9 import os
10 10 import sys
11 11 import errno
12 12 import shutil
13 13 import random
14 14 import glob
15 15 from warnings import warn
16 16 from hashlib import md5
17 17
18 18 from IPython.utils.process import system
19 19 from IPython.utils import py3compat
20 20 from IPython.utils.decorators import undoc
21 21
22 22 #-----------------------------------------------------------------------------
23 23 # Code
24 24 #-----------------------------------------------------------------------------
25 25
26 26 fs_encoding = sys.getfilesystemencoding()
27 27
28 28 def _writable_dir(path):
29 29 """Whether `path` is a directory, to which the user has write access."""
30 30 return os.path.isdir(path) and os.access(path, os.W_OK)
31 31
32 32 if sys.platform == 'win32':
33 33 def _get_long_path_name(path):
34 34 """Get a long path name (expand ~) on Windows using ctypes.
35 35
36 36 Examples
37 37 --------
38 38
39 39 >>> get_long_path_name('c:\\docume~1')
40 40 u'c:\\\\Documents and Settings'
41 41
42 42 """
43 43 try:
44 44 import ctypes
45 45 except ImportError:
46 46 raise ImportError('you need to have ctypes installed for this to work')
47 47 _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
48 48 _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
49 49 ctypes.c_uint ]
50 50
51 51 buf = ctypes.create_unicode_buffer(260)
52 52 rv = _GetLongPathName(path, buf, 260)
53 53 if rv == 0 or rv > 260:
54 54 return path
55 55 else:
56 56 return buf.value
57 57 else:
58 58 def _get_long_path_name(path):
59 59 """Dummy no-op."""
60 60 return path
61 61
62 62
63 63
64 64 def get_long_path_name(path):
65 65 """Expand a path into its long form.
66 66
67 67 On Windows this expands any ~ in the paths. On other platforms, it is
68 68 a null operation.
69 69 """
70 70 return _get_long_path_name(path)
71 71
72 72
73 73 def unquote_filename(name, win32=(sys.platform=='win32')):
74 74 """ On Windows, remove leading and trailing quotes from filenames.
75 75
76 76 This function has been deprecated and should not be used any more:
77 77 unquoting is now taken care of by :func:`IPython.utils.process.arg_split`.
78 78 """
79 79 warn("'unquote_filename' is deprecated since IPython 5.0 and should not "
80 80 "be used anymore", DeprecationWarning, stacklevel=2)
81 81 if win32:
82 82 if name.startswith(("'", '"')) and name.endswith(("'", '"')):
83 83 name = name[1:-1]
84 84 return name
85 85
86 86
87 87 def compress_user(path):
88 88 """Reverse of :func:`os.path.expanduser`
89 89 """
90 90 path = str(path)
91 91 home = os.path.expanduser('~')
92 92 if path.startswith(home):
93 93 path = "~" + path[len(home):]
94 94 return path
95 95
96 96 def get_py_filename(name, force_win32=None):
97 97 """Return a valid python filename in the current directory.
98 98
99 99 If the given name is not a file, it adds '.py' and searches again.
100 100 Raises IOError with an informative message if the file isn't found.
101 101 """
102 102
103 103 name = os.path.expanduser(name)
104 104 if force_win32 is not None:
105 105 warn("The 'force_win32' argument to 'get_py_filename' is deprecated "
106 106 "since IPython 5.0 and should not be used anymore",
107 107 DeprecationWarning, stacklevel=2)
108 108 if not os.path.isfile(name) and not name.endswith('.py'):
109 109 name += '.py'
110 110 if os.path.isfile(name):
111 111 return name
112 112 else:
113 113 raise IOError('File `%r` not found.' % name)
114 114
115 115
116 116 def filefind(filename, path_dirs=None):
117 117 """Find a file by looking through a sequence of paths.
118 118
119 119 This iterates through a sequence of paths looking for a file and returns
120 120 the full, absolute path of the first occurence of the file. If no set of
121 121 path dirs is given, the filename is tested as is, after running through
122 122 :func:`expandvars` and :func:`expanduser`. Thus a simple call::
123 123
124 124 filefind('myfile.txt')
125 125
126 126 will find the file in the current working dir, but::
127 127
128 128 filefind('~/myfile.txt')
129 129
130 130 Will find the file in the users home directory. This function does not
131 131 automatically try any paths, such as the cwd or the user's home directory.
132 132
133 133 Parameters
134 134 ----------
135 135 filename : str
136 136 The filename to look for.
137 137 path_dirs : str, None or sequence of str
138 138 The sequence of paths to look for the file in. If None, the filename
139 139 need to be absolute or be in the cwd. If a string, the string is
140 140 put into a sequence and the searched. If a sequence, walk through
141 141 each element and join with ``filename``, calling :func:`expandvars`
142 142 and :func:`expanduser` before testing for existence.
143 143
144 144 Returns
145 145 -------
146 146 Raises :exc:`IOError` or returns absolute path to file.
147 147 """
148 148
149 149 # If paths are quoted, abspath gets confused, strip them...
150 150 filename = filename.strip('"').strip("'")
151 151 # If the input is an absolute path, just check it exists
152 152 if os.path.isabs(filename) and os.path.isfile(filename):
153 153 return filename
154 154
155 155 if path_dirs is None:
156 156 path_dirs = ("",)
157 157 elif isinstance(path_dirs, str):
158 158 path_dirs = (path_dirs,)
159 159
160 160 for path in path_dirs:
161 if path == '.': path = py3compat.getcwd()
161 if path == '.': path = os.getcwd()
162 162 testname = expand_path(os.path.join(path, filename))
163 163 if os.path.isfile(testname):
164 164 return os.path.abspath(testname)
165 165
166 166 raise IOError("File %r does not exist in any of the search paths: %r" %
167 167 (filename, path_dirs) )
168 168
169 169
170 170 class HomeDirError(Exception):
171 171 pass
172 172
173 173
174 174 def get_home_dir(require_writable=False):
175 175 """Return the 'home' directory, as a unicode string.
176 176
177 177 Uses os.path.expanduser('~'), and checks for writability.
178 178
179 179 See stdlib docs for how this is determined.
180 180 $HOME is first priority on *ALL* platforms.
181 181
182 182 Parameters
183 183 ----------
184 184
185 185 require_writable : bool [default: False]
186 186 if True:
187 187 guarantees the return value is a writable directory, otherwise
188 188 raises HomeDirError
189 189 if False:
190 190 The path is resolved, but it is not guaranteed to exist or be writable.
191 191 """
192 192
193 193 homedir = os.path.expanduser('~')
194 194 # Next line will make things work even when /home/ is a symlink to
195 195 # /usr/home as it is on FreeBSD, for example
196 196 homedir = os.path.realpath(homedir)
197 197
198 198 if not _writable_dir(homedir) and os.name == 'nt':
199 199 # expanduser failed, use the registry to get the 'My Documents' folder.
200 200 try:
201 201 try:
202 202 import winreg as wreg # Py 3
203 203 except ImportError:
204 204 import _winreg as wreg # Py 2
205 205 key = wreg.OpenKey(
206 206 wreg.HKEY_CURRENT_USER,
207 207 "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
208 208 )
209 209 homedir = wreg.QueryValueEx(key,'Personal')[0]
210 210 key.Close()
211 211 except:
212 212 pass
213 213
214 214 if (not require_writable) or _writable_dir(homedir):
215 215 return py3compat.cast_unicode(homedir, fs_encoding)
216 216 else:
217 217 raise HomeDirError('%s is not a writable dir, '
218 218 'set $HOME environment variable to override' % homedir)
219 219
220 220 def get_xdg_dir():
221 221 """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
222 222
223 223 This is only for non-OS X posix (Linux,Unix,etc.) systems.
224 224 """
225 225
226 226 env = os.environ
227 227
228 228 if os.name == 'posix' and sys.platform != 'darwin':
229 229 # Linux, Unix, AIX, etc.
230 230 # use ~/.config if empty OR not set
231 231 xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
232 232 if xdg and _writable_dir(xdg):
233 233 return py3compat.cast_unicode(xdg, fs_encoding)
234 234
235 235 return None
236 236
237 237
238 238 def get_xdg_cache_dir():
239 239 """Return the XDG_CACHE_HOME, if it is defined and exists, else None.
240 240
241 241 This is only for non-OS X posix (Linux,Unix,etc.) systems.
242 242 """
243 243
244 244 env = os.environ
245 245
246 246 if os.name == 'posix' and sys.platform != 'darwin':
247 247 # Linux, Unix, AIX, etc.
248 248 # use ~/.cache if empty OR not set
249 249 xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
250 250 if xdg and _writable_dir(xdg):
251 251 return py3compat.cast_unicode(xdg, fs_encoding)
252 252
253 253 return None
254 254
255 255
256 256 @undoc
257 257 def get_ipython_dir():
258 258 warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
259 259 from IPython.paths import get_ipython_dir
260 260 return get_ipython_dir()
261 261
262 262 @undoc
263 263 def get_ipython_cache_dir():
264 264 warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
265 265 from IPython.paths import get_ipython_cache_dir
266 266 return get_ipython_cache_dir()
267 267
268 268 @undoc
269 269 def get_ipython_package_dir():
270 270 warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
271 271 from IPython.paths import get_ipython_package_dir
272 272 return get_ipython_package_dir()
273 273
274 274 @undoc
275 275 def get_ipython_module_path(module_str):
276 276 warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
277 277 from IPython.paths import get_ipython_module_path
278 278 return get_ipython_module_path(module_str)
279 279
280 280 @undoc
281 281 def locate_profile(profile='default'):
282 282 warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
283 283 from IPython.paths import locate_profile
284 284 return locate_profile(profile=profile)
285 285
286 286 def expand_path(s):
287 287 """Expand $VARS and ~names in a string, like a shell
288 288
289 289 :Examples:
290 290
291 291 In [2]: os.environ['FOO']='test'
292 292
293 293 In [3]: expand_path('variable FOO is $FOO')
294 294 Out[3]: 'variable FOO is test'
295 295 """
296 296 # This is a pretty subtle hack. When expand user is given a UNC path
297 297 # on Windows (\\server\share$\%username%), os.path.expandvars, removes
298 298 # the $ to get (\\server\share\%username%). I think it considered $
299 299 # alone an empty var. But, we need the $ to remains there (it indicates
300 300 # a hidden share).
301 301 if os.name=='nt':
302 302 s = s.replace('$\\', 'IPYTHON_TEMP')
303 303 s = os.path.expandvars(os.path.expanduser(s))
304 304 if os.name=='nt':
305 305 s = s.replace('IPYTHON_TEMP', '$\\')
306 306 return s
307 307
308 308
309 309 def unescape_glob(string):
310 310 """Unescape glob pattern in `string`."""
311 311 def unescape(s):
312 312 for pattern in '*[]!?':
313 313 s = s.replace(r'\{0}'.format(pattern), pattern)
314 314 return s
315 315 return '\\'.join(map(unescape, string.split('\\\\')))
316 316
317 317
318 318 def shellglob(args):
319 319 """
320 320 Do glob expansion for each element in `args` and return a flattened list.
321 321
322 322 Unmatched glob pattern will remain as-is in the returned list.
323 323
324 324 """
325 325 expanded = []
326 326 # Do not unescape backslash in Windows as it is interpreted as
327 327 # path separator:
328 328 unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
329 329 for a in args:
330 330 expanded.extend(glob.glob(a) or [unescape(a)])
331 331 return expanded
332 332
333 333
334 334 def target_outdated(target,deps):
335 335 """Determine whether a target is out of date.
336 336
337 337 target_outdated(target,deps) -> 1/0
338 338
339 339 deps: list of filenames which MUST exist.
340 340 target: single filename which may or may not exist.
341 341
342 342 If target doesn't exist or is older than any file listed in deps, return
343 343 true, otherwise return false.
344 344 """
345 345 try:
346 346 target_time = os.path.getmtime(target)
347 347 except os.error:
348 348 return 1
349 349 for dep in deps:
350 350 dep_time = os.path.getmtime(dep)
351 351 if dep_time > target_time:
352 352 #print "For target",target,"Dep failed:",dep # dbg
353 353 #print "times (dep,tar):",dep_time,target_time # dbg
354 354 return 1
355 355 return 0
356 356
357 357
358 358 def target_update(target,deps,cmd):
359 359 """Update a target with a given command given a list of dependencies.
360 360
361 361 target_update(target,deps,cmd) -> runs cmd if target is outdated.
362 362
363 363 This is just a wrapper around target_outdated() which calls the given
364 364 command if target is outdated."""
365 365
366 366 if target_outdated(target,deps):
367 367 system(cmd)
368 368
369 369 @undoc
370 370 def filehash(path):
371 371 """Make an MD5 hash of a file, ignoring any differences in line
372 372 ending characters."""
373 373 warn("filehash() is deprecated since IPython 4.0", DeprecationWarning, stacklevel=2)
374 374 with open(path, "rU") as f:
375 375 return md5(py3compat.str_to_bytes(f.read())).hexdigest()
376 376
377 377 ENOLINK = 1998
378 378
379 379 def link(src, dst):
380 380 """Hard links ``src`` to ``dst``, returning 0 or errno.
381 381
382 382 Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
383 383 supported by the operating system.
384 384 """
385 385
386 386 if not hasattr(os, "link"):
387 387 return ENOLINK
388 388 link_errno = 0
389 389 try:
390 390 os.link(src, dst)
391 391 except OSError as e:
392 392 link_errno = e.errno
393 393 return link_errno
394 394
395 395
396 396 def link_or_copy(src, dst):
397 397 """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.
398 398
399 399 Attempts to maintain the semantics of ``shutil.copy``.
400 400
401 401 Because ``os.link`` does not overwrite files, a unique temporary file
402 402 will be used if the target already exists, then that file will be moved
403 403 into place.
404 404 """
405 405
406 406 if os.path.isdir(dst):
407 407 dst = os.path.join(dst, os.path.basename(src))
408 408
409 409 link_errno = link(src, dst)
410 410 if link_errno == errno.EEXIST:
411 411 if os.stat(src).st_ino == os.stat(dst).st_ino:
412 412 # dst is already a hard link to the correct file, so we don't need
413 413 # to do anything else. If we try to link and rename the file
414 414 # anyway, we get duplicate files - see http://bugs.python.org/issue21876
415 415 return
416 416
417 417 new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
418 418 try:
419 419 link_or_copy(src, new_dst)
420 420 except:
421 421 try:
422 422 os.remove(new_dst)
423 423 except OSError:
424 424 pass
425 425 raise
426 426 os.rename(new_dst, dst)
427 427 elif link_errno != 0:
428 428 # Either link isn't supported, or the filesystem doesn't support
429 429 # linking, or 'src' and 'dst' are on different filesystems.
430 430 shutil.copy(src, dst)
431 431
432 432 def ensure_dir_exists(path, mode=0o755):
433 433 """ensure that a directory exists
434 434
435 435 If it doesn't exist, try to create it and protect against a race condition
436 436 if another process is doing the same.
437 437
438 438 The default permissions are 755, which differ from os.makedirs default of 777.
439 439 """
440 440 if not os.path.exists(path):
441 441 try:
442 442 os.makedirs(path, mode=mode)
443 443 except OSError as e:
444 444 if e.errno != errno.EEXIST:
445 445 raise
446 446 elif not os.path.isdir(path):
447 447 raise IOError("%r exists but is not a directory" % path)
@@ -1,69 +1,69 b''
1 1 # encoding: utf-8
2 2 """
3 3 Utilities for working with external processes.
4 4 """
5 5
6 6 # Copyright (c) IPython Development Team.
7 7 # Distributed under the terms of the Modified BSD License.
8 8
9 9
10 10 import os
11 11 import sys
12 12
13 13 if sys.platform == 'win32':
14 14 from ._process_win32 import system, getoutput, arg_split, check_pid
15 15 elif sys.platform == 'cli':
16 16 from ._process_cli import system, getoutput, arg_split, check_pid
17 17 else:
18 18 from ._process_posix import system, getoutput, arg_split, check_pid
19 19
20 20 from ._process_common import getoutputerror, get_output_error_code, process_handler
21 21 from . import py3compat
22 22
23 23
24 24 class FindCmdError(Exception):
25 25 pass
26 26
27 27
28 28 def find_cmd(cmd):
29 29 """Find absolute path to executable cmd in a cross platform manner.
30 30
31 31 This function tries to determine the full path to a command line program
32 32 using `which` on Unix/Linux/OS X and `win32api` on Windows. Most of the
33 33 time it will use the version that is first on the users `PATH`.
34 34
35 35 Warning, don't use this to find IPython command line programs as there
36 36 is a risk you will find the wrong one. Instead find those using the
37 37 following code and looking for the application itself::
38 38
39 39 import sys
40 40 argv = [sys.executable, '-m', 'IPython']
41 41
42 42 Parameters
43 43 ----------
44 44 cmd : str
45 45 The command line program to look for.
46 46 """
47 47 path = py3compat.which(cmd)
48 48 if path is None:
49 49 raise FindCmdError('command could not be found: %s' % cmd)
50 50 return path
51 51
52 52
53 53 def abbrev_cwd():
54 54 """ Return abbreviated version of cwd, e.g. d:mydir """
55 cwd = py3compat.getcwd().replace('\\','/')
55 cwd = os.getcwd().replace('\\','/')
56 56 drivepart = ''
57 57 tail = cwd
58 58 if sys.platform == 'win32':
59 59 if len(cwd) < 4:
60 60 return cwd
61 61 drivepart,tail = os.path.splitdrive(cwd)
62 62
63 63
64 64 parts = tail.split('/')
65 65 if len(parts) > 2:
66 66 tail = '/'.join(parts[-2:])
67 67
68 68 return (drivepart + (
69 69 cwd == '/' and '/' or tail))
@@ -1,120 +1,120 b''
1 1 # encoding: utf-8
2 2 """
3 3 Utilities for working with terminals.
4 4
5 5 Authors:
6 6
7 7 * Brian E. Granger
8 8 * Fernando Perez
9 9 * Alexander Belchenko (e-mail: bialix AT ukr.net)
10 10 """
11 11
12 12 # Copyright (c) IPython Development Team.
13 13 # Distributed under the terms of the Modified BSD License.
14 14
15 15 import os
16 16 import sys
17 17 import warnings
18 18 try:
19 19 from shutil import get_terminal_size as _get_terminal_size
20 20 except ImportError:
21 21 # use backport on Python 2
22 22 from backports.shutil_get_terminal_size import get_terminal_size as _get_terminal_size
23 23
24 24 from . import py3compat
25 25
26 26 #-----------------------------------------------------------------------------
27 27 # Code
28 28 #-----------------------------------------------------------------------------
29 29
30 30 # This variable is part of the expected API of the module:
31 31 ignore_termtitle = True
32 32
33 33
34 34
35 35 if os.name == 'posix':
36 36 def _term_clear():
37 37 os.system('clear')
38 38 elif sys.platform == 'win32':
39 39 def _term_clear():
40 40 os.system('cls')
41 41 else:
42 42 def _term_clear():
43 43 pass
44 44
45 45
46 46
47 47 def toggle_set_term_title(val):
48 48 """Control whether set_term_title is active or not.
49 49
50 50 set_term_title() allows writing to the console titlebar. In embedded
51 51 widgets this can cause problems, so this call can be used to toggle it on
52 52 or off as needed.
53 53
54 54 The default state of the module is for the function to be disabled.
55 55
56 56 Parameters
57 57 ----------
58 58 val : bool
59 59 If True, set_term_title() actually writes to the terminal (using the
60 60 appropriate platform-specific module). If False, it is a no-op.
61 61 """
62 62 global ignore_termtitle
63 63 ignore_termtitle = not(val)
64 64
65 65
66 66 def _set_term_title(*args,**kw):
67 67 """Dummy no-op."""
68 68 pass
69 69
70 70
71 71 def _set_term_title_xterm(title):
72 72 """ Change virtual terminal title in xterm-workalikes """
73 73 sys.stdout.write('\033]0;%s\007' % title)
74 74
75 75 if os.name == 'posix':
76 76 TERM = os.environ.get('TERM','')
77 77 if TERM.startswith('xterm'):
78 78 _set_term_title = _set_term_title_xterm
79 79 elif sys.platform == 'win32':
80 80 try:
81 81 import ctypes
82 82
83 83 SetConsoleTitleW = ctypes.windll.kernel32.SetConsoleTitleW
84 84 SetConsoleTitleW.argtypes = [ctypes.c_wchar_p]
85 85
86 86 def _set_term_title(title):
87 87 """Set terminal title using ctypes to access the Win32 APIs."""
88 88 SetConsoleTitleW(title)
89 89 except ImportError:
90 90 def _set_term_title(title):
91 91 """Set terminal title using the 'title' command."""
92 92 global ignore_termtitle
93 93
94 94 try:
95 95 # Cannot be on network share when issuing system commands
96 curr = py3compat.getcwd()
96 curr = os.getcwd()
97 97 os.chdir("C:")
98 98 ret = os.system("title " + title)
99 99 finally:
100 100 os.chdir(curr)
101 101 if ret:
102 102 # non-zero return code signals error, don't try again
103 103 ignore_termtitle = True
104 104
105 105
106 106 def set_term_title(title):
107 107 """Set terminal title using the necessary platform-dependent calls."""
108 108 if ignore_termtitle:
109 109 return
110 110 _set_term_title(title)
111 111
112 112
113 113 def freeze_term_title():
114 114 warnings.warn("This function is deprecated, use toggle_set_term_title()")
115 115 global ignore_termtitle
116 116 ignore_termtitle = True
117 117
118 118
119 119 def get_terminal_size(defaultx=80, defaulty=25):
120 120 return _get_terminal_size((defaultx, defaulty))
@@ -1,496 +1,496 b''
1 1 # encoding: utf-8
2 2 """Tests for IPython.utils.path.py"""
3 3
4 4 # Copyright (c) IPython Development Team.
5 5 # Distributed under the terms of the Modified BSD License.
6 6
7 7 import errno
8 8 import os
9 9 import shutil
10 10 import sys
11 11 import tempfile
12 12 import warnings
13 13 from contextlib import contextmanager
14 14
15 15 try: # Python 3.3+
16 16 from unittest.mock import patch
17 17 except ImportError:
18 18 from mock import patch
19 19
20 20 from os.path import join, abspath, split
21 21
22 22 from nose import SkipTest
23 23 import nose.tools as nt
24 24
25 25 from nose import with_setup
26 26
27 27 import IPython
28 28 from IPython import paths
29 29 from IPython.testing import decorators as dec
30 30 from IPython.testing.decorators import (skip_if_not_win32, skip_win32,
31 31 onlyif_unicode_paths,)
32 32 from IPython.testing.tools import make_tempfile, AssertPrints
33 33 from IPython.utils import path
34 34 from IPython.utils import py3compat
35 35 from IPython.utils.tempdir import TemporaryDirectory
36 36
37 37 # Platform-dependent imports
38 38 try:
39 39 import winreg as wreg # Py 3
40 40 except ImportError:
41 41 try:
42 42 import _winreg as wreg # Py 2
43 43 except ImportError:
44 44 #Fake _winreg module on none windows platforms
45 45 import types
46 46 wr_name = "winreg" if py3compat.PY3 else "_winreg"
47 47 sys.modules[wr_name] = types.ModuleType(wr_name)
48 48 try:
49 49 import winreg as wreg
50 50 except ImportError:
51 51 import _winreg as wreg
52 52 #Add entries that needs to be stubbed by the testing code
53 53 (wreg.OpenKey, wreg.QueryValueEx,) = (None, None)
54 54
55 55 try:
56 56 reload
57 57 except NameError: # Python 3
58 58 from imp import reload
59 59
60 60 #-----------------------------------------------------------------------------
61 61 # Globals
62 62 #-----------------------------------------------------------------------------
63 63 env = os.environ
64 64 TMP_TEST_DIR = tempfile.mkdtemp()
65 65 HOME_TEST_DIR = join(TMP_TEST_DIR, "home_test_dir")
66 66 #
67 67 # Setup/teardown functions/decorators
68 68 #
69 69
70 70 def setup():
71 71 """Setup testenvironment for the module:
72 72
73 73 - Adds dummy home dir tree
74 74 """
75 75 # Do not mask exceptions here. In particular, catching WindowsError is a
76 76 # problem because that exception is only defined on Windows...
77 77 os.makedirs(os.path.join(HOME_TEST_DIR, 'ipython'))
78 78
79 79
80 80 def teardown():
81 81 """Teardown testenvironment for the module:
82 82
83 83 - Remove dummy home dir tree
84 84 """
85 85 # Note: we remove the parent test dir, which is the root of all test
86 86 # subdirs we may have created. Use shutil instead of os.removedirs, so
87 87 # that non-empty directories are all recursively removed.
88 88 shutil.rmtree(TMP_TEST_DIR)
89 89
90 90
91 91 def setup_environment():
92 92 """Setup testenvironment for some functions that are tested
93 93 in this module. In particular this functions stores attributes
94 94 and other things that we need to stub in some test functions.
95 95 This needs to be done on a function level and not module level because
96 96 each testfunction needs a pristine environment.
97 97 """
98 98 global oldstuff, platformstuff
99 99 oldstuff = (env.copy(), os.name, sys.platform, path.get_home_dir, IPython.__file__, os.getcwd())
100 100
101 101 def teardown_environment():
102 102 """Restore things that were remembered by the setup_environment function
103 103 """
104 104 (oldenv, os.name, sys.platform, path.get_home_dir, IPython.__file__, old_wd) = oldstuff
105 105 os.chdir(old_wd)
106 106 reload(path)
107 107
108 108 for key in list(env):
109 109 if key not in oldenv:
110 110 del env[key]
111 111 env.update(oldenv)
112 112 if hasattr(sys, 'frozen'):
113 113 del sys.frozen
114 114
115 115 # Build decorator that uses the setup_environment/setup_environment
116 116 with_environment = with_setup(setup_environment, teardown_environment)
117 117
118 118 @skip_if_not_win32
119 119 @with_environment
120 120 def test_get_home_dir_1():
121 121 """Testcase for py2exe logic, un-compressed lib
122 122 """
123 123 unfrozen = path.get_home_dir()
124 124 sys.frozen = True
125 125
126 126 #fake filename for IPython.__init__
127 127 IPython.__file__ = abspath(join(HOME_TEST_DIR, "Lib/IPython/__init__.py"))
128 128
129 129 home_dir = path.get_home_dir()
130 130 nt.assert_equal(home_dir, unfrozen)
131 131
132 132
133 133 @skip_if_not_win32
134 134 @with_environment
135 135 def test_get_home_dir_2():
136 136 """Testcase for py2exe logic, compressed lib
137 137 """
138 138 unfrozen = path.get_home_dir()
139 139 sys.frozen = True
140 140 #fake filename for IPython.__init__
141 141 IPython.__file__ = abspath(join(HOME_TEST_DIR, "Library.zip/IPython/__init__.py")).lower()
142 142
143 143 home_dir = path.get_home_dir(True)
144 144 nt.assert_equal(home_dir, unfrozen)
145 145
146 146
147 147 @with_environment
148 148 def test_get_home_dir_3():
149 149 """get_home_dir() uses $HOME if set"""
150 150 env["HOME"] = HOME_TEST_DIR
151 151 home_dir = path.get_home_dir(True)
152 152 # get_home_dir expands symlinks
153 153 nt.assert_equal(home_dir, os.path.realpath(env["HOME"]))
154 154
155 155
156 156 @with_environment
157 157 def test_get_home_dir_4():
158 158 """get_home_dir() still works if $HOME is not set"""
159 159
160 160 if 'HOME' in env: del env['HOME']
161 161 # this should still succeed, but we don't care what the answer is
162 162 home = path.get_home_dir(False)
163 163
164 164 @with_environment
165 165 def test_get_home_dir_5():
166 166 """raise HomeDirError if $HOME is specified, but not a writable dir"""
167 167 env['HOME'] = abspath(HOME_TEST_DIR+'garbage')
168 168 # set os.name = posix, to prevent My Documents fallback on Windows
169 169 os.name = 'posix'
170 170 nt.assert_raises(path.HomeDirError, path.get_home_dir, True)
171 171
172 172 # Should we stub wreg fully so we can run the test on all platforms?
173 173 @skip_if_not_win32
174 174 @with_environment
175 175 def test_get_home_dir_8():
176 176 """Using registry hack for 'My Documents', os=='nt'
177 177
178 178 HOMESHARE, HOMEDRIVE, HOMEPATH, USERPROFILE and others are missing.
179 179 """
180 180 os.name = 'nt'
181 181 # Remove from stub environment all keys that may be set
182 182 for key in ['HOME', 'HOMESHARE', 'HOMEDRIVE', 'HOMEPATH', 'USERPROFILE']:
183 183 env.pop(key, None)
184 184
185 185 class key:
186 186 def Close(self):
187 187 pass
188 188
189 189 with patch.object(wreg, 'OpenKey', return_value=key()), \
190 190 patch.object(wreg, 'QueryValueEx', return_value=[abspath(HOME_TEST_DIR)]):
191 191 home_dir = path.get_home_dir()
192 192 nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
193 193
194 194 @with_environment
195 195 def test_get_xdg_dir_0():
196 196 """test_get_xdg_dir_0, check xdg_dir"""
197 197 reload(path)
198 198 path._writable_dir = lambda path: True
199 199 path.get_home_dir = lambda : 'somewhere'
200 200 os.name = "posix"
201 201 sys.platform = "linux2"
202 202 env.pop('IPYTHON_DIR', None)
203 203 env.pop('IPYTHONDIR', None)
204 204 env.pop('XDG_CONFIG_HOME', None)
205 205
206 206 nt.assert_equal(path.get_xdg_dir(), os.path.join('somewhere', '.config'))
207 207
208 208
209 209 @with_environment
210 210 def test_get_xdg_dir_1():
211 211 """test_get_xdg_dir_1, check nonexistant xdg_dir"""
212 212 reload(path)
213 213 path.get_home_dir = lambda : HOME_TEST_DIR
214 214 os.name = "posix"
215 215 sys.platform = "linux2"
216 216 env.pop('IPYTHON_DIR', None)
217 217 env.pop('IPYTHONDIR', None)
218 218 env.pop('XDG_CONFIG_HOME', None)
219 219 nt.assert_equal(path.get_xdg_dir(), None)
220 220
221 221 @with_environment
222 222 def test_get_xdg_dir_2():
223 223 """test_get_xdg_dir_2, check xdg_dir default to ~/.config"""
224 224 reload(path)
225 225 path.get_home_dir = lambda : HOME_TEST_DIR
226 226 os.name = "posix"
227 227 sys.platform = "linux2"
228 228 env.pop('IPYTHON_DIR', None)
229 229 env.pop('IPYTHONDIR', None)
230 230 env.pop('XDG_CONFIG_HOME', None)
231 231 cfgdir=os.path.join(path.get_home_dir(), '.config')
232 232 if not os.path.exists(cfgdir):
233 233 os.makedirs(cfgdir)
234 234
235 235 nt.assert_equal(path.get_xdg_dir(), cfgdir)
236 236
237 237 @with_environment
238 238 def test_get_xdg_dir_3():
239 239 """test_get_xdg_dir_3, check xdg_dir not used on OS X"""
240 240 reload(path)
241 241 path.get_home_dir = lambda : HOME_TEST_DIR
242 242 os.name = "posix"
243 243 sys.platform = "darwin"
244 244 env.pop('IPYTHON_DIR', None)
245 245 env.pop('IPYTHONDIR', None)
246 246 env.pop('XDG_CONFIG_HOME', None)
247 247 cfgdir=os.path.join(path.get_home_dir(), '.config')
248 248 if not os.path.exists(cfgdir):
249 249 os.makedirs(cfgdir)
250 250
251 251 nt.assert_equal(path.get_xdg_dir(), None)
252 252
253 253 def test_filefind():
254 254 """Various tests for filefind"""
255 255 f = tempfile.NamedTemporaryFile()
256 256 # print 'fname:',f.name
257 257 alt_dirs = paths.get_ipython_dir()
258 258 t = path.filefind(f.name, alt_dirs)
259 259 # print 'found:',t
260 260
261 261
262 262 @dec.skip_if_not_win32
263 263 def test_get_long_path_name_win32():
264 264 with TemporaryDirectory() as tmpdir:
265 265
266 266 # Make a long path. Expands the path of tmpdir prematurely as it may already have a long
267 267 # path component, so ensure we include the long form of it
268 268 long_path = os.path.join(path.get_long_path_name(tmpdir), u'this is my long path name')
269 269 os.makedirs(long_path)
270 270
271 271 # Test to see if the short path evaluates correctly.
272 272 short_path = os.path.join(tmpdir, u'THISIS~1')
273 273 evaluated_path = path.get_long_path_name(short_path)
274 274 nt.assert_equal(evaluated_path.lower(), long_path.lower())
275 275
276 276
277 277 @dec.skip_win32
278 278 def test_get_long_path_name():
279 279 p = path.get_long_path_name('/usr/local')
280 280 nt.assert_equal(p,'/usr/local')
281 281
282 282 @dec.skip_win32 # can't create not-user-writable dir on win
283 283 @with_environment
284 284 def test_not_writable_ipdir():
285 285 tmpdir = tempfile.mkdtemp()
286 286 os.name = "posix"
287 287 env.pop('IPYTHON_DIR', None)
288 288 env.pop('IPYTHONDIR', None)
289 289 env.pop('XDG_CONFIG_HOME', None)
290 290 env['HOME'] = tmpdir
291 291 ipdir = os.path.join(tmpdir, '.ipython')
292 292 os.mkdir(ipdir, 0o555)
293 293 try:
294 294 open(os.path.join(ipdir, "_foo_"), 'w').close()
295 295 except IOError:
296 296 pass
297 297 else:
298 298 # I can still write to an unwritable dir,
299 299 # assume I'm root and skip the test
300 300 raise SkipTest("I can't create directories that I can't write to")
301 301 with AssertPrints('is not a writable location', channel='stderr'):
302 302 ipdir = paths.get_ipython_dir()
303 303 env.pop('IPYTHON_DIR', None)
304 304
305 305 @with_environment
306 306 def test_get_py_filename():
307 307 os.chdir(TMP_TEST_DIR)
308 308 with make_tempfile('foo.py'):
309 309 nt.assert_equal(path.get_py_filename('foo.py'), 'foo.py')
310 310 nt.assert_equal(path.get_py_filename('foo'), 'foo.py')
311 311 with make_tempfile('foo'):
312 312 nt.assert_equal(path.get_py_filename('foo'), 'foo')
313 313 nt.assert_raises(IOError, path.get_py_filename, 'foo.py')
314 314 nt.assert_raises(IOError, path.get_py_filename, 'foo')
315 315 nt.assert_raises(IOError, path.get_py_filename, 'foo.py')
316 316 true_fn = 'foo with spaces.py'
317 317 with make_tempfile(true_fn):
318 318 nt.assert_equal(path.get_py_filename('foo with spaces'), true_fn)
319 319 nt.assert_equal(path.get_py_filename('foo with spaces.py'), true_fn)
320 320 nt.assert_raises(IOError, path.get_py_filename, '"foo with spaces.py"')
321 321 nt.assert_raises(IOError, path.get_py_filename, "'foo with spaces.py'")
322 322
323 323 @onlyif_unicode_paths
324 324 def test_unicode_in_filename():
325 325 """When a file doesn't exist, the exception raised should be safe to call
326 326 str() on - i.e. in Python 2 it must only have ASCII characters.
327 327
328 328 https://github.com/ipython/ipython/issues/875
329 329 """
330 330 try:
331 331 # these calls should not throw unicode encode exceptions
332 332 path.get_py_filename(u'fooéè.py', force_win32=False)
333 333 except IOError as ex:
334 334 str(ex)
335 335
336 336
337 337 class TestShellGlob(object):
338 338
339 339 @classmethod
340 340 def setUpClass(cls):
341 341 cls.filenames_start_with_a = ['a0', 'a1', 'a2']
342 342 cls.filenames_end_with_b = ['0b', '1b', '2b']
343 343 cls.filenames = cls.filenames_start_with_a + cls.filenames_end_with_b
344 344 cls.tempdir = TemporaryDirectory()
345 345 td = cls.tempdir.name
346 346
347 347 with cls.in_tempdir():
348 348 # Create empty files
349 349 for fname in cls.filenames:
350 350 open(os.path.join(td, fname), 'w').close()
351 351
352 352 @classmethod
353 353 def tearDownClass(cls):
354 354 cls.tempdir.cleanup()
355 355
356 356 @classmethod
357 357 @contextmanager
358 358 def in_tempdir(cls):
359 save = py3compat.getcwd()
359 save = os.getcwd()
360 360 try:
361 361 os.chdir(cls.tempdir.name)
362 362 yield
363 363 finally:
364 364 os.chdir(save)
365 365
366 366 def check_match(self, patterns, matches):
367 367 with self.in_tempdir():
368 368 # glob returns unordered list. that's why sorted is required.
369 369 nt.assert_equals(sorted(path.shellglob(patterns)),
370 370 sorted(matches))
371 371
372 372 def common_cases(self):
373 373 return [
374 374 (['*'], self.filenames),
375 375 (['a*'], self.filenames_start_with_a),
376 376 (['*c'], ['*c']),
377 377 (['*', 'a*', '*b', '*c'], self.filenames
378 378 + self.filenames_start_with_a
379 379 + self.filenames_end_with_b
380 380 + ['*c']),
381 381 (['a[012]'], self.filenames_start_with_a),
382 382 ]
383 383
384 384 @skip_win32
385 385 def test_match_posix(self):
386 386 for (patterns, matches) in self.common_cases() + [
387 387 ([r'\*'], ['*']),
388 388 ([r'a\*', 'a*'], ['a*'] + self.filenames_start_with_a),
389 389 ([r'a\[012]'], ['a[012]']),
390 390 ]:
391 391 yield (self.check_match, patterns, matches)
392 392
393 393 @skip_if_not_win32
394 394 def test_match_windows(self):
395 395 for (patterns, matches) in self.common_cases() + [
396 396 # In windows, backslash is interpreted as path
397 397 # separator. Therefore, you can't escape glob
398 398 # using it.
399 399 ([r'a\*', 'a*'], [r'a\*'] + self.filenames_start_with_a),
400 400 ([r'a\[012]'], [r'a\[012]']),
401 401 ]:
402 402 yield (self.check_match, patterns, matches)
403 403
404 404
405 405 def test_unescape_glob():
406 406 nt.assert_equals(path.unescape_glob(r'\*\[\!\]\?'), '*[!]?')
407 407 nt.assert_equals(path.unescape_glob(r'\\*'), r'\*')
408 408 nt.assert_equals(path.unescape_glob(r'\\\*'), r'\*')
409 409 nt.assert_equals(path.unescape_glob(r'\\a'), r'\a')
410 410 nt.assert_equals(path.unescape_glob(r'\a'), r'\a')
411 411
412 412
413 413 def test_ensure_dir_exists():
414 414 with TemporaryDirectory() as td:
415 415 d = os.path.join(td, u'βˆ‚ir')
416 416 path.ensure_dir_exists(d) # create it
417 417 assert os.path.isdir(d)
418 418 path.ensure_dir_exists(d) # no-op
419 419 f = os.path.join(td, u'Ζ’ile')
420 420 open(f, 'w').close() # touch
421 421 with nt.assert_raises(IOError):
422 422 path.ensure_dir_exists(f)
423 423
424 424 class TestLinkOrCopy(object):
425 425 def setUp(self):
426 426 self.tempdir = TemporaryDirectory()
427 427 self.src = self.dst("src")
428 428 with open(self.src, "w") as f:
429 429 f.write("Hello, world!")
430 430
431 431 def tearDown(self):
432 432 self.tempdir.cleanup()
433 433
434 434 def dst(self, *args):
435 435 return os.path.join(self.tempdir.name, *args)
436 436
437 437 def assert_inode_not_equal(self, a, b):
438 438 nt.assert_not_equals(os.stat(a).st_ino, os.stat(b).st_ino,
439 439 "%r and %r do reference the same indoes" %(a, b))
440 440
441 441 def assert_inode_equal(self, a, b):
442 442 nt.assert_equals(os.stat(a).st_ino, os.stat(b).st_ino,
443 443 "%r and %r do not reference the same indoes" %(a, b))
444 444
445 445 def assert_content_equal(self, a, b):
446 446 with open(a) as a_f:
447 447 with open(b) as b_f:
448 448 nt.assert_equals(a_f.read(), b_f.read())
449 449
450 450 @skip_win32
451 451 def test_link_successful(self):
452 452 dst = self.dst("target")
453 453 path.link_or_copy(self.src, dst)
454 454 self.assert_inode_equal(self.src, dst)
455 455
456 456 @skip_win32
457 457 def test_link_into_dir(self):
458 458 dst = self.dst("some_dir")
459 459 os.mkdir(dst)
460 460 path.link_or_copy(self.src, dst)
461 461 expected_dst = self.dst("some_dir", os.path.basename(self.src))
462 462 self.assert_inode_equal(self.src, expected_dst)
463 463
464 464 @skip_win32
465 465 def test_target_exists(self):
466 466 dst = self.dst("target")
467 467 open(dst, "w").close()
468 468 path.link_or_copy(self.src, dst)
469 469 self.assert_inode_equal(self.src, dst)
470 470
471 471 @skip_win32
472 472 def test_no_link(self):
473 473 real_link = os.link
474 474 try:
475 475 del os.link
476 476 dst = self.dst("target")
477 477 path.link_or_copy(self.src, dst)
478 478 self.assert_content_equal(self.src, dst)
479 479 self.assert_inode_not_equal(self.src, dst)
480 480 finally:
481 481 os.link = real_link
482 482
483 483 @skip_if_not_win32
484 484 def test_windows(self):
485 485 dst = self.dst("target")
486 486 path.link_or_copy(self.src, dst)
487 487 self.assert_content_equal(self.src, dst)
488 488
489 489 def test_link_twice(self):
490 490 # Linking the same file twice shouldn't leave duplicates around.
491 491 # See https://github.com/ipython/ipython/issues/6450
492 492 dst = self.dst('target')
493 493 path.link_or_copy(self.src, dst)
494 494 path.link_or_copy(self.src, dst)
495 495 self.assert_inode_equal(self.src, dst)
496 496 nt.assert_equal(sorted(os.listdir(self.tempdir.name)), ['src', 'target'])
General Comments 0
You need to be logged in to leave comments. Login now