##// END OF EJS Templates
Merge pull request #13213 from Kojoley/fix-bunch-of-doctests...
Matthias Bussonnier -
r26940:32497c8d merge
parent child Browse files
Show More
@@ -1,672 +1,674 b''
1 1 """Various display related classes.
2 2
3 3 Authors : MinRK, gregcaporaso, dannystaple
4 4 """
5 5 from html import escape as html_escape
6 6 from os.path import exists, isfile, splitext, abspath, join, isdir
7 7 from os import walk, sep, fsdecode
8 8
9 9 from IPython.core.display import DisplayObject, TextDisplayObject
10 10
11 11 from typing import Tuple, Iterable
12 12
13 13 __all__ = ['Audio', 'IFrame', 'YouTubeVideo', 'VimeoVideo', 'ScribdDocument',
14 14 'FileLink', 'FileLinks', 'Code']
15 15
16 16
17 17 class Audio(DisplayObject):
18 18 """Create an audio object.
19 19
20 20 When this object is returned by an input cell or passed to the
21 21 display function, it will result in Audio controls being displayed
22 22 in the frontend (only works in the notebook).
23 23
24 24 Parameters
25 25 ----------
26 26 data : numpy array, list, unicode, str or bytes
27 27 Can be one of
28 28
29 29 * Numpy 1d array containing the desired waveform (mono)
30 30 * Numpy 2d array containing waveforms for each channel.
31 31 Shape=(NCHAN, NSAMPLES). For the standard channel order, see
32 32 http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
33 33 * List of float or integer representing the waveform (mono)
34 34 * String containing the filename
35 35 * Bytestring containing raw PCM data or
36 36 * URL pointing to a file on the web.
37 37
38 38 If the array option is used, the waveform will be normalized.
39 39
40 40 If a filename or url is used, the format support will be browser
41 41 dependent.
42 42 url : unicode
43 43 A URL to download the data from.
44 44 filename : unicode
45 45 Path to a local file to load the data from.
46 46 embed : boolean
47 47 Should the audio data be embedded using a data URI (True) or should
48 48 the original source be referenced. Set this to True if you want the
49 49 audio to playable later with no internet connection in the notebook.
50 50
51 51 Default is `True`, unless the keyword argument `url` is set, then
52 52 default value is `False`.
53 53 rate : integer
54 54 The sampling rate of the raw data.
55 55 Only required when data parameter is being used as an array
56 56 autoplay : bool
57 57 Set to True if the audio should immediately start playing.
58 58 Default is `False`.
59 59 normalize : bool
60 60 Whether audio should be normalized (rescaled) to the maximum possible
61 61 range. Default is `True`. When set to `False`, `data` must be between
62 62 -1 and 1 (inclusive), otherwise an error is raised.
63 63 Applies only when `data` is a list or array of samples; other types of
64 64 audio are never normalized.
65 65
66 66 Examples
67 67 --------
68 68
69 69 Generate a sound
70 70
71 71 >>> import numpy as np
72 ... framerate = 44100
73 ... t = np.linspace(0,5,framerate*5)
74 ... data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t)
75 ... Audio(data, rate=framerate)
72 >>> framerate = 44100
73 >>> t = np.linspace(0,5,framerate*5)
74 >>> data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t)
75 >>> Audio(data, rate=framerate)
76 <IPython.lib.display.Audio object>
76 77
77 78 Can also do stereo or more channels
78 79
79 80 >>> dataleft = np.sin(2*np.pi*220*t)
80 ... dataright = np.sin(2*np.pi*224*t)
81 ... Audio([dataleft, dataright], rate=framerate)
81 >>> dataright = np.sin(2*np.pi*224*t)
82 >>> Audio([dataleft, dataright], rate=framerate)
83 <IPython.lib.display.Audio object>
82 84
83 85 From URL:
84 86
85 >>> Audio("http://www.nch.com.au/acm/8k16bitpcm.wav")
86 >>> Audio(url="http://www.w3schools.com/html/horse.ogg")
87 >>> Audio("http://www.nch.com.au/acm/8k16bitpcm.wav") # doctest: +SKIP
88 >>> Audio(url="http://www.w3schools.com/html/horse.ogg") # doctest: +SKIP
87 89
88 90 From a File:
89 91
90 >>> Audio('/path/to/sound.wav')
91 >>> Audio(filename='/path/to/sound.ogg')
92 >>> Audio('/path/to/sound.wav') # doctest: +SKIP
93 >>> Audio(filename='/path/to/sound.ogg') # doctest: +SKIP
92 94
93 95 From Bytes:
94 96
95 >>> Audio(b'RAW_WAV_DATA..')
96 >>> Audio(data=b'RAW_WAV_DATA..')
97 >>> Audio(b'RAW_WAV_DATA..') # doctest: +SKIP
98 >>> Audio(data=b'RAW_WAV_DATA..') # doctest: +SKIP
97 99
98 100 See Also
99 101 --------
100 102 ipywidgets.Audio
101 103
102 104 AUdio widget with more more flexibility and options.
103 105
104 106 """
105 107 _read_flags = 'rb'
106 108
107 109 def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, autoplay=False, normalize=True, *,
108 110 element_id=None):
109 111 if filename is None and url is None and data is None:
110 112 raise ValueError("No audio data found. Expecting filename, url, or data.")
111 113 if embed is False and url is None:
112 114 raise ValueError("No url found. Expecting url when embed=False")
113 115
114 116 if url is not None and embed is not True:
115 117 self.embed = False
116 118 else:
117 119 self.embed = True
118 120 self.autoplay = autoplay
119 121 self.element_id = element_id
120 122 super(Audio, self).__init__(data=data, url=url, filename=filename)
121 123
122 124 if self.data is not None and not isinstance(self.data, bytes):
123 125 if rate is None:
124 126 raise ValueError("rate must be specified when data is a numpy array or list of audio samples.")
125 127 self.data = Audio._make_wav(data, rate, normalize)
126 128
127 129 def reload(self):
128 130 """Reload the raw data from file or URL."""
129 131 import mimetypes
130 132 if self.embed:
131 133 super(Audio, self).reload()
132 134
133 135 if self.filename is not None:
134 136 self.mimetype = mimetypes.guess_type(self.filename)[0]
135 137 elif self.url is not None:
136 138 self.mimetype = mimetypes.guess_type(self.url)[0]
137 139 else:
138 140 self.mimetype = "audio/wav"
139 141
140 142 @staticmethod
141 143 def _make_wav(data, rate, normalize):
142 144 """ Transform a numpy array to a PCM bytestring """
143 145 from io import BytesIO
144 146 import wave
145 147
146 148 try:
147 149 scaled, nchan = Audio._validate_and_normalize_with_numpy(data, normalize)
148 150 except ImportError:
149 151 scaled, nchan = Audio._validate_and_normalize_without_numpy(data, normalize)
150 152
151 153 fp = BytesIO()
152 154 waveobj = wave.open(fp,mode='wb')
153 155 waveobj.setnchannels(nchan)
154 156 waveobj.setframerate(rate)
155 157 waveobj.setsampwidth(2)
156 158 waveobj.setcomptype('NONE','NONE')
157 159 waveobj.writeframes(scaled)
158 160 val = fp.getvalue()
159 161 waveobj.close()
160 162
161 163 return val
162 164
163 165 @staticmethod
164 166 def _validate_and_normalize_with_numpy(data, normalize) -> Tuple[bytes, int]:
165 167 import numpy as np
166 168
167 169 data = np.array(data, dtype=float)
168 170 if len(data.shape) == 1:
169 171 nchan = 1
170 172 elif len(data.shape) == 2:
171 173 # In wave files,channels are interleaved. E.g.,
172 174 # "L1R1L2R2..." for stereo. See
173 175 # http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
174 176 # for channel ordering
175 177 nchan = data.shape[0]
176 178 data = data.T.ravel()
177 179 else:
178 180 raise ValueError('Array audio input must be a 1D or 2D array')
179 181
180 182 max_abs_value = np.max(np.abs(data))
181 183 normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize)
182 184 scaled = data / normalization_factor * 32767
183 185 return scaled.astype("<h").tobytes(), nchan
184 186
185 187 @staticmethod
186 188 def _validate_and_normalize_without_numpy(data, normalize):
187 189 import array
188 190 import sys
189 191
190 192 data = array.array('f', data)
191 193
192 194 try:
193 195 max_abs_value = float(max([abs(x) for x in data]))
194 196 except TypeError as e:
195 197 raise TypeError('Only lists of mono audio are '
196 198 'supported if numpy is not installed') from e
197 199
198 200 normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize)
199 201 scaled = array.array('h', [int(x / normalization_factor * 32767) for x in data])
200 202 if sys.byteorder == 'big':
201 203 scaled.byteswap()
202 204 nchan = 1
203 205 return scaled.tobytes(), nchan
204 206
205 207 @staticmethod
206 208 def _get_normalization_factor(max_abs_value, normalize):
207 209 if not normalize and max_abs_value > 1:
208 210 raise ValueError('Audio data must be between -1 and 1 when normalize=False.')
209 211 return max_abs_value if normalize else 1
210 212
211 213 def _data_and_metadata(self):
212 214 """shortcut for returning metadata with url information, if defined"""
213 215 md = {}
214 216 if self.url:
215 217 md['url'] = self.url
216 218 if md:
217 219 return self.data, md
218 220 else:
219 221 return self.data
220 222
221 223 def _repr_html_(self):
222 224 src = """
223 225 <audio {element_id} controls="controls" {autoplay}>
224 226 <source src="{src}" type="{type}" />
225 227 Your browser does not support the audio element.
226 228 </audio>
227 229 """
228 230 return src.format(src=self.src_attr(), type=self.mimetype, autoplay=self.autoplay_attr(),
229 231 element_id=self.element_id_attr())
230 232
231 233 def src_attr(self):
232 234 import base64
233 235 if self.embed and (self.data is not None):
234 236 data = base64=base64.b64encode(self.data).decode('ascii')
235 237 return """data:{type};base64,{base64}""".format(type=self.mimetype,
236 238 base64=data)
237 239 elif self.url is not None:
238 240 return self.url
239 241 else:
240 242 return ""
241 243
242 244 def autoplay_attr(self):
243 245 if(self.autoplay):
244 246 return 'autoplay="autoplay"'
245 247 else:
246 248 return ''
247 249
248 250 def element_id_attr(self):
249 251 if (self.element_id):
250 252 return 'id="{element_id}"'.format(element_id=self.element_id)
251 253 else:
252 254 return ''
253 255
254 256 class IFrame(object):
255 257 """
256 258 Generic class to embed an iframe in an IPython notebook
257 259 """
258 260
259 261 iframe = """
260 262 <iframe
261 263 width="{width}"
262 264 height="{height}"
263 265 src="{src}{params}"
264 266 frameborder="0"
265 267 allowfullscreen
266 268 {extras}
267 269 ></iframe>
268 270 """
269 271
270 272 def __init__(self, src, width, height, extras: Iterable[str] = None, **kwargs):
271 273 if extras is None:
272 274 extras = []
273 275
274 276 self.src = src
275 277 self.width = width
276 278 self.height = height
277 279 self.extras = extras
278 280 self.params = kwargs
279 281
280 282 def _repr_html_(self):
281 283 """return the embed iframe"""
282 284 if self.params:
283 285 from urllib.parse import urlencode
284 286 params = "?" + urlencode(self.params)
285 287 else:
286 288 params = ""
287 289 return self.iframe.format(
288 290 src=self.src,
289 291 width=self.width,
290 292 height=self.height,
291 293 params=params,
292 294 extras=" ".join(self.extras),
293 295 )
294 296
295 297
296 298 class YouTubeVideo(IFrame):
297 299 """Class for embedding a YouTube Video in an IPython session, based on its video id.
298 300
299 301 e.g. to embed the video from https://www.youtube.com/watch?v=foo , you would
300 302 do::
301 303
302 304 vid = YouTubeVideo("foo")
303 305 display(vid)
304 306
305 307 To start from 30 seconds::
306 308
307 309 vid = YouTubeVideo("abc", start=30)
308 310 display(vid)
309 311
310 312 To calculate seconds from time as hours, minutes, seconds use
311 313 :class:`datetime.timedelta`::
312 314
313 315 start=int(timedelta(hours=1, minutes=46, seconds=40).total_seconds())
314 316
315 317 Other parameters can be provided as documented at
316 318 https://developers.google.com/youtube/player_parameters#Parameters
317 319
318 320 When converting the notebook using nbconvert, a jpeg representation of the video
319 321 will be inserted in the document.
320 322 """
321 323
322 324 def __init__(self, id, width=400, height=300, allow_autoplay=False, **kwargs):
323 325 self.id=id
324 326 src = "https://www.youtube.com/embed/{0}".format(id)
325 327 if allow_autoplay:
326 328 extras = list(kwargs.get("extras", [])) + ['allow="autoplay"']
327 329 kwargs.update(autoplay=1, extras=extras)
328 330 super(YouTubeVideo, self).__init__(src, width, height, **kwargs)
329 331
330 332 def _repr_jpeg_(self):
331 333 # Deferred import
332 334 from urllib.request import urlopen
333 335
334 336 try:
335 337 return urlopen("https://img.youtube.com/vi/{id}/hqdefault.jpg".format(id=self.id)).read()
336 338 except IOError:
337 339 return None
338 340
339 341 class VimeoVideo(IFrame):
340 342 """
341 343 Class for embedding a Vimeo video in an IPython session, based on its video id.
342 344 """
343 345
344 346 def __init__(self, id, width=400, height=300, **kwargs):
345 347 src="https://player.vimeo.com/video/{0}".format(id)
346 348 super(VimeoVideo, self).__init__(src, width, height, **kwargs)
347 349
348 350 class ScribdDocument(IFrame):
349 351 """
350 352 Class for embedding a Scribd document in an IPython session
351 353
352 354 Use the start_page params to specify a starting point in the document
353 355 Use the view_mode params to specify display type one off scroll | slideshow | book
354 356
355 357 e.g to Display Wes' foundational paper about PANDAS in book mode from page 3
356 358
357 359 ScribdDocument(71048089, width=800, height=400, start_page=3, view_mode="book")
358 360 """
359 361
360 362 def __init__(self, id, width=400, height=300, **kwargs):
361 363 src="https://www.scribd.com/embeds/{0}/content".format(id)
362 364 super(ScribdDocument, self).__init__(src, width, height, **kwargs)
363 365
364 366 class FileLink(object):
365 367 """Class for embedding a local file link in an IPython session, based on path
366 368
367 369 e.g. to embed a link that was generated in the IPython notebook as my/data.txt
368 370
369 371 you would do::
370 372
371 373 local_file = FileLink("my/data.txt")
372 374 display(local_file)
373 375
374 376 or in the HTML notebook, just::
375 377
376 378 FileLink("my/data.txt")
377 379 """
378 380
379 381 html_link_str = "<a href='%s' target='_blank'>%s</a>"
380 382
381 383 def __init__(self,
382 384 path,
383 385 url_prefix='',
384 386 result_html_prefix='',
385 387 result_html_suffix='<br>'):
386 388 """
387 389 Parameters
388 390 ----------
389 391 path : str
390 392 path to the file or directory that should be formatted
391 393 url_prefix : str
392 394 prefix to be prepended to all files to form a working link [default:
393 395 '']
394 396 result_html_prefix : str
395 397 text to append to beginning to link [default: '']
396 398 result_html_suffix : str
397 399 text to append at the end of link [default: '<br>']
398 400 """
399 401 if isdir(path):
400 402 raise ValueError("Cannot display a directory using FileLink. "
401 403 "Use FileLinks to display '%s'." % path)
402 404 self.path = fsdecode(path)
403 405 self.url_prefix = url_prefix
404 406 self.result_html_prefix = result_html_prefix
405 407 self.result_html_suffix = result_html_suffix
406 408
407 409 def _format_path(self):
408 410 fp = ''.join([self.url_prefix, html_escape(self.path)])
409 411 return ''.join([self.result_html_prefix,
410 412 self.html_link_str % \
411 413 (fp, html_escape(self.path, quote=False)),
412 414 self.result_html_suffix])
413 415
414 416 def _repr_html_(self):
415 417 """return html link to file
416 418 """
417 419 if not exists(self.path):
418 420 return ("Path (<tt>%s</tt>) doesn't exist. "
419 421 "It may still be in the process of "
420 422 "being generated, or you may have the "
421 423 "incorrect path." % self.path)
422 424
423 425 return self._format_path()
424 426
425 427 def __repr__(self):
426 428 """return absolute path to file
427 429 """
428 430 return abspath(self.path)
429 431
430 432 class FileLinks(FileLink):
431 433 """Class for embedding local file links in an IPython session, based on path
432 434
433 435 e.g. to embed links to files that were generated in the IPython notebook
434 436 under ``my/data``, you would do::
435 437
436 438 local_files = FileLinks("my/data")
437 439 display(local_files)
438 440
439 441 or in the HTML notebook, just::
440 442
441 443 FileLinks("my/data")
442 444 """
443 445 def __init__(self,
444 446 path,
445 447 url_prefix='',
446 448 included_suffixes=None,
447 449 result_html_prefix='',
448 450 result_html_suffix='<br>',
449 451 notebook_display_formatter=None,
450 452 terminal_display_formatter=None,
451 453 recursive=True):
452 454 """
453 455 See :class:`FileLink` for the ``path``, ``url_prefix``,
454 456 ``result_html_prefix`` and ``result_html_suffix`` parameters.
455 457
456 458 included_suffixes : list
457 459 Filename suffixes to include when formatting output [default: include
458 460 all files]
459 461
460 462 notebook_display_formatter : function
461 463 Used to format links for display in the notebook. See discussion of
462 464 formatter functions below.
463 465
464 466 terminal_display_formatter : function
465 467 Used to format links for display in the terminal. See discussion of
466 468 formatter functions below.
467 469
468 470 Formatter functions must be of the form::
469 471
470 472 f(dirname, fnames, included_suffixes)
471 473
472 474 dirname : str
473 475 The name of a directory
474 476 fnames : list
475 477 The files in that directory
476 478 included_suffixes : list
477 479 The file suffixes that should be included in the output (passing None
478 480 meansto include all suffixes in the output in the built-in formatters)
479 481 recursive : boolean
480 482 Whether to recurse into subdirectories. Default is True.
481 483
482 484 The function should return a list of lines that will be printed in the
483 485 notebook (if passing notebook_display_formatter) or the terminal (if
484 486 passing terminal_display_formatter). This function is iterated over for
485 487 each directory in self.path. Default formatters are in place, can be
486 488 passed here to support alternative formatting.
487 489
488 490 """
489 491 if isfile(path):
490 492 raise ValueError("Cannot display a file using FileLinks. "
491 493 "Use FileLink to display '%s'." % path)
492 494 self.included_suffixes = included_suffixes
493 495 # remove trailing slashes for more consistent output formatting
494 496 path = path.rstrip('/')
495 497
496 498 self.path = path
497 499 self.url_prefix = url_prefix
498 500 self.result_html_prefix = result_html_prefix
499 501 self.result_html_suffix = result_html_suffix
500 502
501 503 self.notebook_display_formatter = \
502 504 notebook_display_formatter or self._get_notebook_display_formatter()
503 505 self.terminal_display_formatter = \
504 506 terminal_display_formatter or self._get_terminal_display_formatter()
505 507
506 508 self.recursive = recursive
507 509
508 510 def _get_display_formatter(self,
509 511 dirname_output_format,
510 512 fname_output_format,
511 513 fp_format,
512 514 fp_cleaner=None):
513 515 """ generate built-in formatter function
514 516
515 517 this is used to define both the notebook and terminal built-in
516 518 formatters as they only differ by some wrapper text for each entry
517 519
518 520 dirname_output_format: string to use for formatting directory
519 521 names, dirname will be substituted for a single "%s" which
520 522 must appear in this string
521 523 fname_output_format: string to use for formatting file names,
522 524 if a single "%s" appears in the string, fname will be substituted
523 525 if two "%s" appear in the string, the path to fname will be
524 526 substituted for the first and fname will be substituted for the
525 527 second
526 528 fp_format: string to use for formatting filepaths, must contain
527 529 exactly two "%s" and the dirname will be substituted for the first
528 530 and fname will be substituted for the second
529 531 """
530 532 def f(dirname, fnames, included_suffixes=None):
531 533 result = []
532 534 # begin by figuring out which filenames, if any,
533 535 # are going to be displayed
534 536 display_fnames = []
535 537 for fname in fnames:
536 538 if (isfile(join(dirname,fname)) and
537 539 (included_suffixes is None or
538 540 splitext(fname)[1] in included_suffixes)):
539 541 display_fnames.append(fname)
540 542
541 543 if len(display_fnames) == 0:
542 544 # if there are no filenames to display, don't print anything
543 545 # (not even the directory name)
544 546 pass
545 547 else:
546 548 # otherwise print the formatted directory name followed by
547 549 # the formatted filenames
548 550 dirname_output_line = dirname_output_format % dirname
549 551 result.append(dirname_output_line)
550 552 for fname in display_fnames:
551 553 fp = fp_format % (dirname,fname)
552 554 if fp_cleaner is not None:
553 555 fp = fp_cleaner(fp)
554 556 try:
555 557 # output can include both a filepath and a filename...
556 558 fname_output_line = fname_output_format % (fp, fname)
557 559 except TypeError:
558 560 # ... or just a single filepath
559 561 fname_output_line = fname_output_format % fname
560 562 result.append(fname_output_line)
561 563 return result
562 564 return f
563 565
564 566 def _get_notebook_display_formatter(self,
565 567 spacer="&nbsp;&nbsp;"):
566 568 """ generate function to use for notebook formatting
567 569 """
568 570 dirname_output_format = \
569 571 self.result_html_prefix + "%s/" + self.result_html_suffix
570 572 fname_output_format = \
571 573 self.result_html_prefix + spacer + self.html_link_str + self.result_html_suffix
572 574 fp_format = self.url_prefix + '%s/%s'
573 575 if sep == "\\":
574 576 # Working on a platform where the path separator is "\", so
575 577 # must convert these to "/" for generating a URI
576 578 def fp_cleaner(fp):
577 579 # Replace all occurrences of backslash ("\") with a forward
578 580 # slash ("/") - this is necessary on windows when a path is
579 581 # provided as input, but we must link to a URI
580 582 return fp.replace('\\','/')
581 583 else:
582 584 fp_cleaner = None
583 585
584 586 return self._get_display_formatter(dirname_output_format,
585 587 fname_output_format,
586 588 fp_format,
587 589 fp_cleaner)
588 590
589 591 def _get_terminal_display_formatter(self,
590 592 spacer=" "):
591 593 """ generate function to use for terminal formatting
592 594 """
593 595 dirname_output_format = "%s/"
594 596 fname_output_format = spacer + "%s"
595 597 fp_format = '%s/%s'
596 598
597 599 return self._get_display_formatter(dirname_output_format,
598 600 fname_output_format,
599 601 fp_format)
600 602
601 603 def _format_path(self):
602 604 result_lines = []
603 605 if self.recursive:
604 606 walked_dir = list(walk(self.path))
605 607 else:
606 608 walked_dir = [next(walk(self.path))]
607 609 walked_dir.sort()
608 610 for dirname, subdirs, fnames in walked_dir:
609 611 result_lines += self.notebook_display_formatter(dirname, fnames, self.included_suffixes)
610 612 return '\n'.join(result_lines)
611 613
612 614 def __repr__(self):
613 615 """return newline-separated absolute paths
614 616 """
615 617 result_lines = []
616 618 if self.recursive:
617 619 walked_dir = list(walk(self.path))
618 620 else:
619 621 walked_dir = [next(walk(self.path))]
620 622 walked_dir.sort()
621 623 for dirname, subdirs, fnames in walked_dir:
622 624 result_lines += self.terminal_display_formatter(dirname, fnames, self.included_suffixes)
623 625 return '\n'.join(result_lines)
624 626
625 627
626 628 class Code(TextDisplayObject):
627 629 """Display syntax-highlighted source code.
628 630
629 631 This uses Pygments to highlight the code for HTML and Latex output.
630 632
631 633 Parameters
632 634 ----------
633 635 data : str
634 636 The code as a string
635 637 url : str
636 638 A URL to fetch the code from
637 639 filename : str
638 640 A local filename to load the code from
639 641 language : str
640 642 The short name of a Pygments lexer to use for highlighting.
641 643 If not specified, it will guess the lexer based on the filename
642 644 or the code. Available lexers: http://pygments.org/docs/lexers/
643 645 """
644 646 def __init__(self, data=None, url=None, filename=None, language=None):
645 647 self.language = language
646 648 super().__init__(data=data, url=url, filename=filename)
647 649
648 650 def _get_lexer(self):
649 651 if self.language:
650 652 from pygments.lexers import get_lexer_by_name
651 653 return get_lexer_by_name(self.language)
652 654 elif self.filename:
653 655 from pygments.lexers import get_lexer_for_filename
654 656 return get_lexer_for_filename(self.filename)
655 657 else:
656 658 from pygments.lexers import guess_lexer
657 659 return guess_lexer(self.data)
658 660
659 661 def __repr__(self):
660 662 return self.data
661 663
662 664 def _repr_html_(self):
663 665 from pygments import highlight
664 666 from pygments.formatters import HtmlFormatter
665 667 fmt = HtmlFormatter()
666 668 style = '<style>{}</style>'.format(fmt.get_style_defs('.output_html'))
667 669 return style + highlight(self.data, self._get_lexer(), fmt)
668 670
669 671 def _repr_latex_(self):
670 672 from pygments import highlight
671 673 from pygments.formatters import LatexFormatter
672 674 return highlight(self.data, self._get_lexer(), LatexFormatter())
@@ -1,532 +1,526 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 Defines a variety of Pygments lexers for highlighting IPython code.
4 4
5 5 This includes:
6 6
7 7 IPythonLexer, IPython3Lexer
8 8 Lexers for pure IPython (python + magic/shell commands)
9 9
10 10 IPythonPartialTracebackLexer, IPythonTracebackLexer
11 11 Supports 2.x and 3.x via keyword `python3`. The partial traceback
12 12 lexer reads everything but the Python code appearing in a traceback.
13 13 The full lexer combines the partial lexer with an IPython lexer.
14 14
15 15 IPythonConsoleLexer
16 16 A lexer for IPython console sessions, with support for tracebacks.
17 17
18 18 IPyLexer
19 19 A friendly lexer which examines the first line of text and from it,
20 20 decides whether to use an IPython lexer or an IPython console lexer.
21 21 This is probably the only lexer that needs to be explicitly added
22 22 to Pygments.
23 23
24 24 """
25 25 #-----------------------------------------------------------------------------
26 26 # Copyright (c) 2013, the IPython Development Team.
27 27 #
28 28 # Distributed under the terms of the Modified BSD License.
29 29 #
30 30 # The full license is in the file COPYING.txt, distributed with this software.
31 31 #-----------------------------------------------------------------------------
32 32
33 33 # Standard library
34 34 import re
35 35
36 36 # Third party
37 37 from pygments.lexers import (
38 38 BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, PythonLexer,
39 39 Python3Lexer, TexLexer)
40 40 from pygments.lexer import (
41 41 Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
42 42 )
43 43 from pygments.token import (
44 44 Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
45 45 )
46 46 from pygments.util import get_bool_opt
47 47
48 48 # Local
49 49
50 50 line_re = re.compile('.*?\n')
51 51
52 52 __all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer',
53 53 'IPythonPartialTracebackLexer', 'IPythonTracebackLexer',
54 54 'IPythonConsoleLexer', 'IPyLexer']
55 55
56 56
57 57 def build_ipy_lexer(python3):
58 58 """Builds IPython lexers depending on the value of `python3`.
59 59
60 60 The lexer inherits from an appropriate Python lexer and then adds
61 61 information about IPython specific keywords (i.e. magic commands,
62 62 shell commands, etc.)
63 63
64 64 Parameters
65 65 ----------
66 66 python3 : bool
67 67 If `True`, then build an IPython lexer from a Python 3 lexer.
68 68
69 69 """
70 70 # It would be nice to have a single IPython lexer class which takes
71 71 # a boolean `python3`. But since there are two Python lexer classes,
72 72 # we will also have two IPython lexer classes.
73 73 if python3:
74 74 PyLexer = Python3Lexer
75 75 name = 'IPython3'
76 76 aliases = ['ipython3']
77 77 doc = """IPython3 Lexer"""
78 78 else:
79 79 PyLexer = PythonLexer
80 80 name = 'IPython'
81 81 aliases = ['ipython2', 'ipython']
82 82 doc = """IPython Lexer"""
83 83
84 84 ipython_tokens = [
85 85 (r'(?s)(\s*)(%%capture)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
86 86 (r'(?s)(\s*)(%%debug)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
87 87 (r'(?is)(\s*)(%%html)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(HtmlLexer))),
88 88 (r'(?s)(\s*)(%%javascript)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
89 89 (r'(?s)(\s*)(%%js)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
90 90 (r'(?s)(\s*)(%%latex)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(TexLexer))),
91 91 (r'(?s)(\s*)(%%perl)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PerlLexer))),
92 92 (r'(?s)(\s*)(%%prun)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
93 93 (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
94 94 (r'(?s)(\s*)(%%python)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
95 95 (r'(?s)(\s*)(%%python2)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PythonLexer))),
96 96 (r'(?s)(\s*)(%%python3)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))),
97 97 (r'(?s)(\s*)(%%ruby)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(RubyLexer))),
98 98 (r'(?s)(\s*)(%%time)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
99 99 (r'(?s)(\s*)(%%timeit)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
100 100 (r'(?s)(\s*)(%%writefile)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
101 101 (r'(?s)(\s*)(%%file)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
102 102 (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
103 103 (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
104 104 (r"(%%?)(\w+)(\?\??)$", bygroups(Operator, Keyword, Operator)),
105 105 (r"\b(\?\??)(\s*)$", bygroups(Operator, Text)),
106 106 (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
107 107 using(BashLexer), Text)),
108 108 (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
109 109 (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
110 110 (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
111 111 (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
112 112 (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)),
113 113 ]
114 114
115 115 tokens = PyLexer.tokens.copy()
116 116 tokens['root'] = ipython_tokens + tokens['root']
117 117
118 118 attrs = {'name': name, 'aliases': aliases, 'filenames': [],
119 119 '__doc__': doc, 'tokens': tokens}
120 120
121 121 return type(name, (PyLexer,), attrs)
122 122
123 123
124 124 IPython3Lexer = build_ipy_lexer(python3=True)
125 125 IPythonLexer = build_ipy_lexer(python3=False)
126 126
127 127
128 128 class IPythonPartialTracebackLexer(RegexLexer):
129 129 """
130 130 Partial lexer for IPython tracebacks.
131 131
132 132 Handles all the non-python output.
133 133
134 134 """
135 135 name = 'IPython Partial Traceback'
136 136
137 137 tokens = {
138 138 'root': [
139 139 # Tracebacks for syntax errors have a different style.
140 140 # For both types of tracebacks, we mark the first line with
141 141 # Generic.Traceback. For syntax errors, we mark the filename
142 142 # as we mark the filenames for non-syntax tracebacks.
143 143 #
144 144 # These two regexps define how IPythonConsoleLexer finds a
145 145 # traceback.
146 146 #
147 147 ## Non-syntax traceback
148 148 (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
149 149 ## Syntax traceback
150 150 (r'^( File)(.*)(, line )(\d+\n)',
151 151 bygroups(Generic.Traceback, Name.Namespace,
152 152 Generic.Traceback, Literal.Number.Integer)),
153 153
154 154 # (Exception Identifier)(Whitespace)(Traceback Message)
155 155 (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
156 156 bygroups(Name.Exception, Generic.Whitespace, Text)),
157 157 # (Module/Filename)(Text)(Callee)(Function Signature)
158 158 # Better options for callee and function signature?
159 159 (r'(.*)( in )(.*)(\(.*\)\n)',
160 160 bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
161 161 # Regular line: (Whitespace)(Line Number)(Python Code)
162 162 (r'(\s*?)(\d+)(.*?\n)',
163 163 bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
164 164 # Emphasized line: (Arrow)(Line Number)(Python Code)
165 165 # Using Exception token so arrow color matches the Exception.
166 166 (r'(-*>?\s?)(\d+)(.*?\n)',
167 167 bygroups(Name.Exception, Literal.Number.Integer, Other)),
168 168 # (Exception Identifier)(Message)
169 169 (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
170 170 bygroups(Name.Exception, Text)),
171 171 # Tag everything else as Other, will be handled later.
172 172 (r'.*\n', Other),
173 173 ],
174 174 }
175 175
176 176
177 177 class IPythonTracebackLexer(DelegatingLexer):
178 178 """
179 179 IPython traceback lexer.
180 180
181 181 For doctests, the tracebacks can be snipped as much as desired with the
182 182 exception to the lines that designate a traceback. For non-syntax error
183 183 tracebacks, this is the line of hyphens. For syntax error tracebacks,
184 184 this is the line which lists the File and line number.
185 185
186 186 """
187 187 # The lexer inherits from DelegatingLexer. The "root" lexer is an
188 188 # appropriate IPython lexer, which depends on the value of the boolean
189 189 # `python3`. First, we parse with the partial IPython traceback lexer.
190 190 # Then, any code marked with the "Other" token is delegated to the root
191 191 # lexer.
192 192 #
193 193 name = 'IPython Traceback'
194 194 aliases = ['ipythontb']
195 195
196 196 def __init__(self, **options):
197 197 self.python3 = get_bool_opt(options, 'python3', False)
198 198 if self.python3:
199 199 self.aliases = ['ipython3tb']
200 200 else:
201 201 self.aliases = ['ipython2tb', 'ipythontb']
202 202
203 203 if self.python3:
204 204 IPyLexer = IPython3Lexer
205 205 else:
206 206 IPyLexer = IPythonLexer
207 207
208 208 DelegatingLexer.__init__(self, IPyLexer,
209 209 IPythonPartialTracebackLexer, **options)
210 210
211 211 class IPythonConsoleLexer(Lexer):
212 212 """
213 213 An IPython console lexer for IPython code-blocks and doctests, such as:
214 214
215 215 .. code-block:: rst
216 216
217 217 .. code-block:: ipythonconsole
218 218
219 219 In [1]: a = 'foo'
220 220
221 221 In [2]: a
222 222 Out[2]: 'foo'
223 223
224 In [3]: print a
224 In [3]: print(a)
225 225 foo
226 226
227 In [4]: 1 / 0
228
229 227
230 228 Support is also provided for IPython exceptions:
231 229
232 230 .. code-block:: rst
233 231
234 232 .. code-block:: ipythonconsole
235 233
236 234 In [1]: raise Exception
237
238 ---------------------------------------------------------------------------
239 Exception Traceback (most recent call last)
240 <ipython-input-1-fca2ab0ca76b> in <module>
241 ----> 1 raise Exception
242
243 Exception:
235 Traceback (most recent call last):
236 ...
237 Exception
244 238
245 239 """
246 240 name = 'IPython console session'
247 241 aliases = ['ipythonconsole']
248 242 mimetypes = ['text/x-ipython-console']
249 243
250 244 # The regexps used to determine what is input and what is output.
251 245 # The default prompts for IPython are:
252 246 #
253 247 # in = 'In [#]: '
254 248 # continuation = ' .D.: '
255 249 # template = 'Out[#]: '
256 250 #
257 251 # Where '#' is the 'prompt number' or 'execution count' and 'D'
258 252 # D is a number of dots matching the width of the execution count
259 253 #
260 254 in1_regex = r'In \[[0-9]+\]: '
261 255 in2_regex = r' \.\.+\.: '
262 256 out_regex = r'Out\[[0-9]+\]: '
263 257
264 258 #: The regex to determine when a traceback starts.
265 259 ipytb_start = re.compile(r'^(\^C)?(-+\n)|^( File)(.*)(, line )(\d+\n)')
266 260
267 261 def __init__(self, **options):
268 262 """Initialize the IPython console lexer.
269 263
270 264 Parameters
271 265 ----------
272 266 python3 : bool
273 267 If `True`, then the console inputs are parsed using a Python 3
274 268 lexer. Otherwise, they are parsed using a Python 2 lexer.
275 269 in1_regex : RegexObject
276 270 The compiled regular expression used to detect the start
277 271 of inputs. Although the IPython configuration setting may have a
278 272 trailing whitespace, do not include it in the regex. If `None`,
279 273 then the default input prompt is assumed.
280 274 in2_regex : RegexObject
281 275 The compiled regular expression used to detect the continuation
282 276 of inputs. Although the IPython configuration setting may have a
283 277 trailing whitespace, do not include it in the regex. If `None`,
284 278 then the default input prompt is assumed.
285 279 out_regex : RegexObject
286 280 The compiled regular expression used to detect outputs. If `None`,
287 281 then the default output prompt is assumed.
288 282
289 283 """
290 284 self.python3 = get_bool_opt(options, 'python3', False)
291 285 if self.python3:
292 286 self.aliases = ['ipython3console']
293 287 else:
294 288 self.aliases = ['ipython2console', 'ipythonconsole']
295 289
296 290 in1_regex = options.get('in1_regex', self.in1_regex)
297 291 in2_regex = options.get('in2_regex', self.in2_regex)
298 292 out_regex = options.get('out_regex', self.out_regex)
299 293
300 294 # So that we can work with input and output prompts which have been
301 295 # rstrip'd (possibly by editors) we also need rstrip'd variants. If
302 296 # we do not do this, then such prompts will be tagged as 'output'.
303 297 # The reason can't just use the rstrip'd variants instead is because
304 298 # we want any whitespace associated with the prompt to be inserted
305 299 # with the token. This allows formatted code to be modified so as hide
306 300 # the appearance of prompts, with the whitespace included. One example
307 301 # use of this is in copybutton.js from the standard lib Python docs.
308 302 in1_regex_rstrip = in1_regex.rstrip() + '\n'
309 303 in2_regex_rstrip = in2_regex.rstrip() + '\n'
310 304 out_regex_rstrip = out_regex.rstrip() + '\n'
311 305
312 306 # Compile and save them all.
313 307 attrs = ['in1_regex', 'in2_regex', 'out_regex',
314 308 'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
315 309 for attr in attrs:
316 310 self.__setattr__(attr, re.compile(locals()[attr]))
317 311
318 312 Lexer.__init__(self, **options)
319 313
320 314 if self.python3:
321 315 pylexer = IPython3Lexer
322 316 tblexer = IPythonTracebackLexer
323 317 else:
324 318 pylexer = IPythonLexer
325 319 tblexer = IPythonTracebackLexer
326 320
327 321 self.pylexer = pylexer(**options)
328 322 self.tblexer = tblexer(**options)
329 323
330 324 self.reset()
331 325
332 326 def reset(self):
333 327 self.mode = 'output'
334 328 self.index = 0
335 329 self.buffer = u''
336 330 self.insertions = []
337 331
338 332 def buffered_tokens(self):
339 333 """
340 334 Generator of unprocessed tokens after doing insertions and before
341 335 changing to a new state.
342 336
343 337 """
344 338 if self.mode == 'output':
345 339 tokens = [(0, Generic.Output, self.buffer)]
346 340 elif self.mode == 'input':
347 341 tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
348 342 else: # traceback
349 343 tokens = self.tblexer.get_tokens_unprocessed(self.buffer)
350 344
351 345 for i, t, v in do_insertions(self.insertions, tokens):
352 346 # All token indexes are relative to the buffer.
353 347 yield self.index + i, t, v
354 348
355 349 # Clear it all
356 350 self.index += len(self.buffer)
357 351 self.buffer = u''
358 352 self.insertions = []
359 353
360 354 def get_mci(self, line):
361 355 """
362 356 Parses the line and returns a 3-tuple: (mode, code, insertion).
363 357
364 358 `mode` is the next mode (or state) of the lexer, and is always equal
365 359 to 'input', 'output', or 'tb'.
366 360
367 361 `code` is a portion of the line that should be added to the buffer
368 362 corresponding to the next mode and eventually lexed by another lexer.
369 363 For example, `code` could be Python code if `mode` were 'input'.
370 364
371 365 `insertion` is a 3-tuple (index, token, text) representing an
372 366 unprocessed "token" that will be inserted into the stream of tokens
373 367 that are created from the buffer once we change modes. This is usually
374 368 the input or output prompt.
375 369
376 370 In general, the next mode depends on current mode and on the contents
377 371 of `line`.
378 372
379 373 """
380 374 # To reduce the number of regex match checks, we have multiple
381 375 # 'if' blocks instead of 'if-elif' blocks.
382 376
383 377 # Check for possible end of input
384 378 in2_match = self.in2_regex.match(line)
385 379 in2_match_rstrip = self.in2_regex_rstrip.match(line)
386 380 if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
387 381 in2_match_rstrip:
388 382 end_input = True
389 383 else:
390 384 end_input = False
391 385 if end_input and self.mode != 'tb':
392 386 # Only look for an end of input when not in tb mode.
393 387 # An ellipsis could appear within the traceback.
394 388 mode = 'output'
395 389 code = u''
396 390 insertion = (0, Generic.Prompt, line)
397 391 return mode, code, insertion
398 392
399 393 # Check for output prompt
400 394 out_match = self.out_regex.match(line)
401 395 out_match_rstrip = self.out_regex_rstrip.match(line)
402 396 if out_match or out_match_rstrip:
403 397 mode = 'output'
404 398 if out_match:
405 399 idx = out_match.end()
406 400 else:
407 401 idx = out_match_rstrip.end()
408 402 code = line[idx:]
409 403 # Use the 'heading' token for output. We cannot use Generic.Error
410 404 # since it would conflict with exceptions.
411 405 insertion = (0, Generic.Heading, line[:idx])
412 406 return mode, code, insertion
413 407
414 408
415 409 # Check for input or continuation prompt (non stripped version)
416 410 in1_match = self.in1_regex.match(line)
417 411 if in1_match or (in2_match and self.mode != 'tb'):
418 412 # New input or when not in tb, continued input.
419 413 # We do not check for continued input when in tb since it is
420 414 # allowable to replace a long stack with an ellipsis.
421 415 mode = 'input'
422 416 if in1_match:
423 417 idx = in1_match.end()
424 418 else: # in2_match
425 419 idx = in2_match.end()
426 420 code = line[idx:]
427 421 insertion = (0, Generic.Prompt, line[:idx])
428 422 return mode, code, insertion
429 423
430 424 # Check for input or continuation prompt (stripped version)
431 425 in1_match_rstrip = self.in1_regex_rstrip.match(line)
432 426 if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
433 427 # New input or when not in tb, continued input.
434 428 # We do not check for continued input when in tb since it is
435 429 # allowable to replace a long stack with an ellipsis.
436 430 mode = 'input'
437 431 if in1_match_rstrip:
438 432 idx = in1_match_rstrip.end()
439 433 else: # in2_match
440 434 idx = in2_match_rstrip.end()
441 435 code = line[idx:]
442 436 insertion = (0, Generic.Prompt, line[:idx])
443 437 return mode, code, insertion
444 438
445 439 # Check for traceback
446 440 if self.ipytb_start.match(line):
447 441 mode = 'tb'
448 442 code = line
449 443 insertion = None
450 444 return mode, code, insertion
451 445
452 446 # All other stuff...
453 447 if self.mode in ('input', 'output'):
454 448 # We assume all other text is output. Multiline input that
455 449 # does not use the continuation marker cannot be detected.
456 450 # For example, the 3 in the following is clearly output:
457 451 #
458 452 # In [1]: print 3
459 453 # 3
460 454 #
461 455 # But the following second line is part of the input:
462 456 #
463 457 # In [2]: while True:
464 458 # print True
465 459 #
466 460 # In both cases, the 2nd line will be 'output'.
467 461 #
468 462 mode = 'output'
469 463 else:
470 464 mode = 'tb'
471 465
472 466 code = line
473 467 insertion = None
474 468
475 469 return mode, code, insertion
476 470
477 471 def get_tokens_unprocessed(self, text):
478 472 self.reset()
479 473 for match in line_re.finditer(text):
480 474 line = match.group()
481 475 mode, code, insertion = self.get_mci(line)
482 476
483 477 if mode != self.mode:
484 478 # Yield buffered tokens before transitioning to new mode.
485 479 for token in self.buffered_tokens():
486 480 yield token
487 481 self.mode = mode
488 482
489 483 if insertion:
490 484 self.insertions.append((len(self.buffer), [insertion]))
491 485 self.buffer += code
492 486
493 487 for token in self.buffered_tokens():
494 488 yield token
495 489
496 490 class IPyLexer(Lexer):
497 491 r"""
498 492 Primary lexer for all IPython-like code.
499 493
500 494 This is a simple helper lexer. If the first line of the text begins with
501 495 "In \[[0-9]+\]:", then the entire text is parsed with an IPython console
502 496 lexer. If not, then the entire text is parsed with an IPython lexer.
503 497
504 498 The goal is to reduce the number of lexers that are registered
505 499 with Pygments.
506 500
507 501 """
508 502 name = 'IPy session'
509 503 aliases = ['ipy']
510 504
511 505 def __init__(self, **options):
512 506 self.python3 = get_bool_opt(options, 'python3', False)
513 507 if self.python3:
514 508 self.aliases = ['ipy3']
515 509 else:
516 510 self.aliases = ['ipy2', 'ipy']
517 511
518 512 Lexer.__init__(self, **options)
519 513
520 514 self.IPythonLexer = IPythonLexer(**options)
521 515 self.IPythonConsoleLexer = IPythonConsoleLexer(**options)
522 516
523 517 def get_tokens_unprocessed(self, text):
524 518 # Search for the input prompt anywhere...this allows code blocks to
525 519 # begin with comments as well.
526 520 if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL):
527 521 lex = self.IPythonConsoleLexer
528 522 else:
529 523 lex = self.IPythonLexer
530 524 for token in lex.get_tokens_unprocessed(text):
531 525 yield token
532 526
@@ -1,114 +1,114 b''
1 1 """
2 2 Password generation for the IPython notebook.
3 3 """
4 4 #-----------------------------------------------------------------------------
5 5 # Imports
6 6 #-----------------------------------------------------------------------------
7 7 # Stdlib
8 8 import getpass
9 9 import hashlib
10 10 import random
11 11
12 12 # Our own
13 13 from IPython.core.error import UsageError
14 14 from IPython.utils.py3compat import encode
15 15
16 16 #-----------------------------------------------------------------------------
17 17 # Globals
18 18 #-----------------------------------------------------------------------------
19 19
20 20 # Length of the salt in nr of hex chars, which implies salt_len * 4
21 21 # bits of randomness.
22 22 salt_len = 12
23 23
24 24 #-----------------------------------------------------------------------------
25 25 # Functions
26 26 #-----------------------------------------------------------------------------
27 27
28 28 def passwd(passphrase=None, algorithm='sha1'):
29 29 """Generate hashed password and salt for use in notebook configuration.
30 30
31 31 In the notebook configuration, set `c.NotebookApp.password` to
32 32 the generated string.
33 33
34 34 Parameters
35 35 ----------
36 36 passphrase : str
37 37 Password to hash. If unspecified, the user is asked to input
38 38 and verify a password.
39 39 algorithm : str
40 40 Hashing algorithm to use (e.g, 'sha1' or any argument supported
41 41 by :func:`hashlib.new`).
42 42
43 43 Returns
44 44 -------
45 45 hashed_passphrase : str
46 46 Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'.
47 47
48 48 Examples
49 49 --------
50 50 >>> passwd('mypassword')
51 'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12'
51 'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12' # random
52 52
53 53 """
54 54 if passphrase is None:
55 55 for i in range(3):
56 56 p0 = getpass.getpass('Enter password: ')
57 57 p1 = getpass.getpass('Verify password: ')
58 58 if p0 == p1:
59 59 passphrase = p0
60 60 break
61 61 else:
62 62 print('Passwords do not match.')
63 63 else:
64 64 raise UsageError('No matching passwords found. Giving up.')
65 65
66 66 h = hashlib.new(algorithm)
67 67 salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len)
68 68 h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii'))
69 69
70 70 return ':'.join((algorithm, salt, h.hexdigest()))
71 71
72 72
73 73 def passwd_check(hashed_passphrase, passphrase):
74 74 """Verify that a given passphrase matches its hashed version.
75 75
76 76 Parameters
77 77 ----------
78 78 hashed_passphrase : str
79 79 Hashed password, in the format returned by `passwd`.
80 80 passphrase : str
81 81 Passphrase to validate.
82 82
83 83 Returns
84 84 -------
85 85 valid : bool
86 86 True if the passphrase matches the hash.
87 87
88 88 Examples
89 89 --------
90 90 >>> from IPython.lib.security import passwd_check
91 91 >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
92 92 ... 'mypassword')
93 93 True
94 94
95 95 >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
96 96 ... 'anotherpassword')
97 97 False
98 98 """
99 99 try:
100 100 algorithm, salt, pw_digest = hashed_passphrase.split(':', 2)
101 101 except (ValueError, TypeError):
102 102 return False
103 103
104 104 try:
105 105 h = hashlib.new(algorithm)
106 106 except ValueError:
107 107 return False
108 108
109 109 if len(pw_digest) == 0:
110 110 return False
111 111
112 112 h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii'))
113 113
114 114 return h.hexdigest() == pw_digest
@@ -1,157 +1,167 b''
1 1 """Simple example using doctests.
2 2
3 3 This file just contains doctests both using plain python and IPython prompts.
4 4 All tests should be loaded by nose.
5 5 """
6 6
7 import os
8
9
7 10 def pyfunc():
8 11 """Some pure python tests...
9 12
10 13 >>> pyfunc()
11 14 'pyfunc'
12 15
13 16 >>> import os
14 17
15 18 >>> 2+3
16 19 5
17 20
18 21 >>> for i in range(3):
19 22 ... print(i, end=' ')
20 23 ... print(i+1, end=' ')
21 24 ...
22 25 0 1 1 2 2 3
23 26 """
24 27 return 'pyfunc'
25 28
26 29 def ipfunc():
27 30 """Some ipython tests...
28 31
29 32 In [1]: import os
30 33
31 34 In [3]: 2+3
32 35 Out[3]: 5
33 36
34 37 In [26]: for i in range(3):
35 38 ....: print(i, end=' ')
36 39 ....: print(i+1, end=' ')
37 40 ....:
38 41 0 1 1 2 2 3
39 42
40 43
41 Examples that access the operating system work:
42
43 In [1]: !echo hello
44 hello
45
46 In [2]: !echo hello > /tmp/foo_iptest
47
48 In [3]: !cat /tmp/foo_iptest
49 hello
50
51 In [4]: rm -f /tmp/foo_iptest
52
53 44 It's OK to use '_' for the last result, but do NOT try to use IPython's
54 45 numbered history of _NN outputs, since those won't exist under the
55 46 doctest environment:
56 47
57 48 In [7]: 'hi'
58 49 Out[7]: 'hi'
59 50
60 51 In [8]: print(repr(_))
61 52 'hi'
62 53
63 54 In [7]: 3+4
64 55 Out[7]: 7
65 56
66 57 In [8]: _+3
67 58 Out[8]: 10
68 59
69 60 In [9]: ipfunc()
70 61 Out[9]: 'ipfunc'
71 62 """
72 63 return 'ipfunc'
73 64
74 65
66 def ipos():
67 """Examples that access the operating system work:
68
69 In [1]: !echo hello
70 hello
71
72 In [2]: !echo hello > /tmp/foo_iptest
73
74 In [3]: !cat /tmp/foo_iptest
75 hello
76
77 In [4]: rm -f /tmp/foo_iptest
78 """
79 pass
80
81
82 ipos.__skip_doctest__ = os.name == "nt"
83
84
75 85 def ranfunc():
76 86 """A function with some random output.
77 87
78 88 Normal examples are verified as usual:
79 89 >>> 1+3
80 90 4
81 91
82 92 But if you put '# random' in the output, it is ignored:
83 93 >>> 1+3
84 94 junk goes here... # random
85 95
86 96 >>> 1+2
87 97 again, anything goes #random
88 98 if multiline, the random mark is only needed once.
89 99
90 100 >>> 1+2
91 101 You can also put the random marker at the end:
92 102 # random
93 103
94 104 >>> 1+2
95 105 # random
96 106 .. or at the beginning.
97 107
98 108 More correct input is properly verified:
99 109 >>> ranfunc()
100 110 'ranfunc'
101 111 """
102 112 return 'ranfunc'
103 113
104 114
105 115 def random_all():
106 116 """A function where we ignore the output of ALL examples.
107 117
108 118 Examples:
109 119
110 120 # all-random
111 121
112 122 This mark tells the testing machinery that all subsequent examples should
113 123 be treated as random (ignoring their output). They are still executed,
114 124 so if a they raise an error, it will be detected as such, but their
115 125 output is completely ignored.
116 126
117 127 >>> 1+3
118 128 junk goes here...
119 129
120 130 >>> 1+3
121 131 klasdfj;
122 132
123 133 >>> 1+2
124 134 again, anything goes
125 135 blah...
126 136 """
127 137 pass
128 138
129 139 def iprand():
130 140 """Some ipython tests with random output.
131 141
132 142 In [7]: 3+4
133 143 Out[7]: 7
134 144
135 145 In [8]: print('hello')
136 146 world # random
137 147
138 148 In [9]: iprand()
139 149 Out[9]: 'iprand'
140 150 """
141 151 return 'iprand'
142 152
143 153 def iprand_all():
144 154 """Some ipython tests with fully random output.
145 155
146 156 # all-random
147 157
148 158 In [7]: 1
149 159 Out[7]: 99
150 160
151 161 In [8]: print('hello')
152 162 world
153 163
154 164 In [9]: iprand_all()
155 165 Out[9]: 'junk'
156 166 """
157 167 return 'iprand_all'
@@ -1,30 +1,30 b''
1 1 =================================
2 2 Tests in example form - IPython
3 3 =================================
4 4
5 5 You can write text files with examples that use IPython prompts (as long as you
6 6 use the nose ipython doctest plugin), but you can not mix and match prompt
7 7 styles in a single file. That is, you either use all ``>>>`` prompts or all
8 8 IPython-style prompts. Your test suite *can* have both types, you just need to
9 9 put each type of example in a separate. Using IPython prompts, you can paste
10 10 directly from your session::
11 11
12 12 In [5]: s="Hello World"
13 13
14 14 In [6]: s.upper()
15 15 Out[6]: 'HELLO WORLD'
16 16
17 17 Another example::
18 18
19 19 In [8]: 1+3
20 20 Out[8]: 4
21 21
22 22 Just like in IPython docstrings, you can use all IPython syntax and features::
23 23
24 In [9]: !echo "hello"
24 In [9]: !echo hello
25 25 hello
26 26
27 27 In [10]: a='hi'
28 28
29 29 In [11]: !echo $a
30 30 hi
@@ -1,164 +1,168 b''
1 1 """Tests for the decorators we've created for IPython.
2 2 """
3 3
4 4 # Module imports
5 5 # Std lib
6 6 import inspect
7 7 import sys
8 8
9 9 # Third party
10 10 import nose.tools as nt
11 11
12 12 # Our own
13 13 from IPython.testing import decorators as dec
14 from IPython.testing.skipdoctest import skip_doctest
14 15
15 16 #-----------------------------------------------------------------------------
16 17 # Utilities
17 18
18 19 # Note: copied from OInspect, kept here so the testing stuff doesn't create
19 20 # circular dependencies and is easier to reuse.
20 21 def getargspec(obj):
21 22 """Get the names and default values of a function's arguments.
22 23
23 24 A tuple of four things is returned: (args, varargs, varkw, defaults).
24 25 'args' is a list of the argument names (it may contain nested lists).
25 26 'varargs' and 'varkw' are the names of the * and ** arguments or None.
26 27 'defaults' is an n-tuple of the default values of the last n arguments.
27 28
28 29 Modified version of inspect.getargspec from the Python Standard
29 30 Library."""
30 31
31 32 if inspect.isfunction(obj):
32 33 func_obj = obj
33 34 elif inspect.ismethod(obj):
34 35 func_obj = obj.__func__
35 36 else:
36 37 raise TypeError('arg is not a Python function')
37 38 args, varargs, varkw = inspect.getargs(func_obj.__code__)
38 39 return args, varargs, varkw, func_obj.__defaults__
39 40
40 41 #-----------------------------------------------------------------------------
41 42 # Testing functions
42 43
43 44 @dec.as_unittest
44 45 def trivial():
45 46 """A trivial test"""
46 47 pass
47 48
48 49
49 50 @dec.skip()
50 51 def test_deliberately_broken():
51 52 """A deliberately broken test - we want to skip this one."""
52 53 1/0
53 54
54 55 @dec.skip('Testing the skip decorator')
55 56 def test_deliberately_broken2():
56 57 """Another deliberately broken test - we want to skip this one."""
57 58 1/0
58 59
59 60
60 61 # Verify that we can correctly skip the doctest for a function at will, but
61 62 # that the docstring itself is NOT destroyed by the decorator.
63 @skip_doctest
62 64 def doctest_bad(x,y=1,**k):
63 65 """A function whose doctest we need to skip.
64 66
65 67 >>> 1+1
66 68 3
67 69 """
68 70 print('x:',x)
69 71 print('y:',y)
70 72 print('k:',k)
71 73
72 74
73 75 def call_doctest_bad():
74 76 """Check that we can still call the decorated functions.
75 77
76 78 >>> doctest_bad(3,y=4)
77 79 x: 3
78 80 y: 4
79 81 k: {}
80 82 """
81 83 pass
82 84
83 85
84 86 def test_skip_dt_decorator():
85 87 """Doctest-skipping decorator should preserve the docstring.
86 88 """
87 89 # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
88 90 check = """A function whose doctest we need to skip.
89 91
90 92 >>> 1+1
91 93 3
92 94 """
93 95 # Fetch the docstring from doctest_bad after decoration.
94 96 val = doctest_bad.__doc__
95 97
96 98 nt.assert_equal(check,val,"doctest_bad docstrings don't match")
97 99
98 100
99 101 # Doctest skipping should work for class methods too
100 102 class FooClass(object):
101 103 """FooClass
102 104
103 105 Example:
104 106
105 107 >>> 1+1
106 108 2
107 109 """
108 110
111 @skip_doctest
109 112 def __init__(self,x):
110 113 """Make a FooClass.
111 114
112 115 Example:
113 116
114 117 >>> f = FooClass(3)
115 118 junk
116 119 """
117 120 print('Making a FooClass.')
118 121 self.x = x
119 122
123 @skip_doctest
120 124 def bar(self,y):
121 125 """Example:
122 126
123 127 >>> ff = FooClass(3)
124 128 >>> ff.bar(0)
125 129 boom!
126 130 >>> 1/0
127 131 bam!
128 132 """
129 133 return 1/y
130 134
131 135 def baz(self,y):
132 136 """Example:
133 137
134 138 >>> ff2 = FooClass(3)
135 139 Making a FooClass.
136 140 >>> ff2.baz(3)
137 141 True
138 142 """
139 143 return self.x==y
140 144
141 145
142 146 def test_skip_dt_decorator2():
143 147 """Doctest-skipping decorator should preserve function signature.
144 148 """
145 149 # Hardcoded correct answer
146 150 dtargs = (['x', 'y'], None, 'k', (1,))
147 151 # Introspect out the value
148 152 dtargsr = getargspec(doctest_bad)
149 153 assert dtargsr==dtargs, \
150 154 "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
151 155
152 156
153 157 @dec.skip_linux
154 158 def test_linux():
155 159 nt.assert_false(sys.platform.startswith('linux'),"This test can't run under linux")
156 160
157 161 @dec.skip_win32
158 162 def test_win32():
159 163 nt.assert_not_equal(sys.platform,'win32',"This test can't run under windows")
160 164
161 165 @dec.skip_osx
162 166 def test_osx():
163 167 nt.assert_not_equal(sys.platform,'darwin',"This test can't run under osx")
164 168
@@ -1,440 +1,440 b''
1 1 # encoding: utf-8
2 2 """
3 3 Utilities for path handling.
4 4 """
5 5
6 6 # Copyright (c) IPython Development Team.
7 7 # Distributed under the terms of the Modified BSD License.
8 8
9 9 import os
10 10 import sys
11 11 import errno
12 12 import shutil
13 13 import random
14 14 import glob
15 15 from warnings import warn
16 16
17 17 from IPython.utils.process import system
18 18 from IPython.utils.decorators import undoc
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Code
22 22 #-----------------------------------------------------------------------------
23 23 fs_encoding = sys.getfilesystemencoding()
24 24
25 25 def _writable_dir(path):
26 26 """Whether `path` is a directory, to which the user has write access."""
27 27 return os.path.isdir(path) and os.access(path, os.W_OK)
28 28
29 29 if sys.platform == 'win32':
30 30 def _get_long_path_name(path):
31 31 """Get a long path name (expand ~) on Windows using ctypes.
32 32
33 33 Examples
34 34 --------
35 35
36 >>> get_long_path_name('c:\\docume~1')
36 >>> get_long_path_name('c:\\\\docume~1')
37 37 'c:\\\\Documents and Settings'
38 38
39 39 """
40 40 try:
41 41 import ctypes
42 42 except ImportError as e:
43 43 raise ImportError('you need to have ctypes installed for this to work') from e
44 44 _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
45 45 _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
46 46 ctypes.c_uint ]
47 47
48 48 buf = ctypes.create_unicode_buffer(260)
49 49 rv = _GetLongPathName(path, buf, 260)
50 50 if rv == 0 or rv > 260:
51 51 return path
52 52 else:
53 53 return buf.value
54 54 else:
55 55 def _get_long_path_name(path):
56 56 """Dummy no-op."""
57 57 return path
58 58
59 59
60 60
61 61 def get_long_path_name(path):
62 62 """Expand a path into its long form.
63 63
64 64 On Windows this expands any ~ in the paths. On other platforms, it is
65 65 a null operation.
66 66 """
67 67 return _get_long_path_name(path)
68 68
69 69
70 70 def unquote_filename(name, win32=(sys.platform=='win32')):
71 71 """ On Windows, remove leading and trailing quotes from filenames.
72 72
73 73 This function has been deprecated and should not be used any more:
74 74 unquoting is now taken care of by :func:`IPython.utils.process.arg_split`.
75 75 """
76 76 warn("'unquote_filename' is deprecated since IPython 5.0 and should not "
77 77 "be used anymore", DeprecationWarning, stacklevel=2)
78 78 if win32:
79 79 if name.startswith(("'", '"')) and name.endswith(("'", '"')):
80 80 name = name[1:-1]
81 81 return name
82 82
83 83
84 84 def compress_user(path):
85 85 """Reverse of :func:`os.path.expanduser`
86 86 """
87 87 home = os.path.expanduser('~')
88 88 if path.startswith(home):
89 89 path = "~" + path[len(home):]
90 90 return path
91 91
92 92 def get_py_filename(name, force_win32=None):
93 93 """Return a valid python filename in the current directory.
94 94
95 95 If the given name is not a file, it adds '.py' and searches again.
96 96 Raises IOError with an informative message if the file isn't found.
97 97 """
98 98
99 99 name = os.path.expanduser(name)
100 100 if force_win32 is not None:
101 101 warn("The 'force_win32' argument to 'get_py_filename' is deprecated "
102 102 "since IPython 5.0 and should not be used anymore",
103 103 DeprecationWarning, stacklevel=2)
104 104 if not os.path.isfile(name) and not name.endswith('.py'):
105 105 name += '.py'
106 106 if os.path.isfile(name):
107 107 return name
108 108 else:
109 109 raise IOError('File `%r` not found.' % name)
110 110
111 111
112 112 def filefind(filename: str, path_dirs=None) -> str:
113 113 """Find a file by looking through a sequence of paths.
114 114
115 115 This iterates through a sequence of paths looking for a file and returns
116 116 the full, absolute path of the first occurrence of the file. If no set of
117 117 path dirs is given, the filename is tested as is, after running through
118 118 :func:`expandvars` and :func:`expanduser`. Thus a simple call::
119 119
120 120 filefind('myfile.txt')
121 121
122 122 will find the file in the current working dir, but::
123 123
124 124 filefind('~/myfile.txt')
125 125
126 126 Will find the file in the users home directory. This function does not
127 127 automatically try any paths, such as the cwd or the user's home directory.
128 128
129 129 Parameters
130 130 ----------
131 131 filename : str
132 132 The filename to look for.
133 133 path_dirs : str, None or sequence of str
134 134 The sequence of paths to look for the file in. If None, the filename
135 135 need to be absolute or be in the cwd. If a string, the string is
136 136 put into a sequence and the searched. If a sequence, walk through
137 137 each element and join with ``filename``, calling :func:`expandvars`
138 138 and :func:`expanduser` before testing for existence.
139 139
140 140 Returns
141 141 -------
142 142 path : str
143 143 returns absolute path to file.
144 144
145 145 Raises
146 146 ------
147 147 IOError
148 148 """
149 149
150 150 # If paths are quoted, abspath gets confused, strip them...
151 151 filename = filename.strip('"').strip("'")
152 152 # If the input is an absolute path, just check it exists
153 153 if os.path.isabs(filename) and os.path.isfile(filename):
154 154 return filename
155 155
156 156 if path_dirs is None:
157 157 path_dirs = ("",)
158 158 elif isinstance(path_dirs, str):
159 159 path_dirs = (path_dirs,)
160 160
161 161 for path in path_dirs:
162 162 if path == '.': path = os.getcwd()
163 163 testname = expand_path(os.path.join(path, filename))
164 164 if os.path.isfile(testname):
165 165 return os.path.abspath(testname)
166 166
167 167 raise IOError("File %r does not exist in any of the search paths: %r" %
168 168 (filename, path_dirs) )
169 169
170 170
171 171 class HomeDirError(Exception):
172 172 pass
173 173
174 174
175 175 def get_home_dir(require_writable=False) -> str:
176 176 """Return the 'home' directory, as a unicode string.
177 177
178 178 Uses os.path.expanduser('~'), and checks for writability.
179 179
180 180 See stdlib docs for how this is determined.
181 181 For Python <3.8, $HOME is first priority on *ALL* platforms.
182 182 For Python >=3.8 on Windows, %HOME% is no longer considered.
183 183
184 184 Parameters
185 185 ----------
186 186 require_writable : bool [default: False]
187 187 if True:
188 188 guarantees the return value is a writable directory, otherwise
189 189 raises HomeDirError
190 190 if False:
191 191 The path is resolved, but it is not guaranteed to exist or be writable.
192 192 """
193 193
194 194 homedir = os.path.expanduser('~')
195 195 # Next line will make things work even when /home/ is a symlink to
196 196 # /usr/home as it is on FreeBSD, for example
197 197 homedir = os.path.realpath(homedir)
198 198
199 199 if not _writable_dir(homedir) and os.name == 'nt':
200 200 # expanduser failed, use the registry to get the 'My Documents' folder.
201 201 try:
202 202 import winreg as wreg
203 203 with wreg.OpenKey(
204 204 wreg.HKEY_CURRENT_USER,
205 205 r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
206 206 ) as key:
207 207 homedir = wreg.QueryValueEx(key,'Personal')[0]
208 208 except:
209 209 pass
210 210
211 211 if (not require_writable) or _writable_dir(homedir):
212 212 assert isinstance(homedir, str), "Homedir should be unicode not bytes"
213 213 return homedir
214 214 else:
215 215 raise HomeDirError('%s is not a writable dir, '
216 216 'set $HOME environment variable to override' % homedir)
217 217
218 218 def get_xdg_dir():
219 219 """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
220 220
221 221 This is only for non-OS X posix (Linux,Unix,etc.) systems.
222 222 """
223 223
224 224 env = os.environ
225 225
226 226 if os.name == 'posix' and sys.platform != 'darwin':
227 227 # Linux, Unix, AIX, etc.
228 228 # use ~/.config if empty OR not set
229 229 xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
230 230 if xdg and _writable_dir(xdg):
231 231 assert isinstance(xdg, str)
232 232 return xdg
233 233
234 234 return None
235 235
236 236
237 237 def get_xdg_cache_dir():
238 238 """Return the XDG_CACHE_HOME, if it is defined and exists, else None.
239 239
240 240 This is only for non-OS X posix (Linux,Unix,etc.) systems.
241 241 """
242 242
243 243 env = os.environ
244 244
245 245 if os.name == 'posix' and sys.platform != 'darwin':
246 246 # Linux, Unix, AIX, etc.
247 247 # use ~/.cache if empty OR not set
248 248 xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
249 249 if xdg and _writable_dir(xdg):
250 250 assert isinstance(xdg, str)
251 251 return xdg
252 252
253 253 return None
254 254
255 255
256 256 @undoc
257 257 def get_ipython_dir():
258 258 warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
259 259 from IPython.paths import get_ipython_dir
260 260 return get_ipython_dir()
261 261
262 262 @undoc
263 263 def get_ipython_cache_dir():
264 264 warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
265 265 from IPython.paths import get_ipython_cache_dir
266 266 return get_ipython_cache_dir()
267 267
268 268 @undoc
269 269 def get_ipython_package_dir():
270 270 warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
271 271 from IPython.paths import get_ipython_package_dir
272 272 return get_ipython_package_dir()
273 273
274 274 @undoc
275 275 def get_ipython_module_path(module_str):
276 276 warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
277 277 from IPython.paths import get_ipython_module_path
278 278 return get_ipython_module_path(module_str)
279 279
280 280 @undoc
281 281 def locate_profile(profile='default'):
282 282 warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
283 283 from IPython.paths import locate_profile
284 284 return locate_profile(profile=profile)
285 285
286 286 def expand_path(s):
287 287 """Expand $VARS and ~names in a string, like a shell
288 288
289 289 :Examples:
290 290
291 291 In [2]: os.environ['FOO']='test'
292 292
293 293 In [3]: expand_path('variable FOO is $FOO')
294 294 Out[3]: 'variable FOO is test'
295 295 """
296 296 # This is a pretty subtle hack. When expand user is given a UNC path
297 297 # on Windows (\\server\share$\%username%), os.path.expandvars, removes
298 298 # the $ to get (\\server\share\%username%). I think it considered $
299 299 # alone an empty var. But, we need the $ to remains there (it indicates
300 300 # a hidden share).
301 301 if os.name=='nt':
302 302 s = s.replace('$\\', 'IPYTHON_TEMP')
303 303 s = os.path.expandvars(os.path.expanduser(s))
304 304 if os.name=='nt':
305 305 s = s.replace('IPYTHON_TEMP', '$\\')
306 306 return s
307 307
308 308
309 309 def unescape_glob(string):
310 310 """Unescape glob pattern in `string`."""
311 311 def unescape(s):
312 312 for pattern in '*[]!?':
313 313 s = s.replace(r'\{0}'.format(pattern), pattern)
314 314 return s
315 315 return '\\'.join(map(unescape, string.split('\\\\')))
316 316
317 317
318 318 def shellglob(args):
319 319 """
320 320 Do glob expansion for each element in `args` and return a flattened list.
321 321
322 322 Unmatched glob pattern will remain as-is in the returned list.
323 323
324 324 """
325 325 expanded = []
326 326 # Do not unescape backslash in Windows as it is interpreted as
327 327 # path separator:
328 328 unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
329 329 for a in args:
330 330 expanded.extend(glob.glob(a) or [unescape(a)])
331 331 return expanded
332 332
333 333
334 334 def target_outdated(target,deps):
335 335 """Determine whether a target is out of date.
336 336
337 337 target_outdated(target,deps) -> 1/0
338 338
339 339 deps: list of filenames which MUST exist.
340 340 target: single filename which may or may not exist.
341 341
342 342 If target doesn't exist or is older than any file listed in deps, return
343 343 true, otherwise return false.
344 344 """
345 345 try:
346 346 target_time = os.path.getmtime(target)
347 347 except os.error:
348 348 return 1
349 349 for dep in deps:
350 350 dep_time = os.path.getmtime(dep)
351 351 if dep_time > target_time:
352 352 #print "For target",target,"Dep failed:",dep # dbg
353 353 #print "times (dep,tar):",dep_time,target_time # dbg
354 354 return 1
355 355 return 0
356 356
357 357
358 358 def target_update(target,deps,cmd):
359 359 """Update a target with a given command given a list of dependencies.
360 360
361 361 target_update(target,deps,cmd) -> runs cmd if target is outdated.
362 362
363 363 This is just a wrapper around target_outdated() which calls the given
364 364 command if target is outdated."""
365 365
366 366 if target_outdated(target,deps):
367 367 system(cmd)
368 368
369 369
370 370 ENOLINK = 1998
371 371
372 372 def link(src, dst):
373 373 """Hard links ``src`` to ``dst``, returning 0 or errno.
374 374
375 375 Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
376 376 supported by the operating system.
377 377 """
378 378
379 379 if not hasattr(os, "link"):
380 380 return ENOLINK
381 381 link_errno = 0
382 382 try:
383 383 os.link(src, dst)
384 384 except OSError as e:
385 385 link_errno = e.errno
386 386 return link_errno
387 387
388 388
389 389 def link_or_copy(src, dst):
390 390 """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.
391 391
392 392 Attempts to maintain the semantics of ``shutil.copy``.
393 393
394 394 Because ``os.link`` does not overwrite files, a unique temporary file
395 395 will be used if the target already exists, then that file will be moved
396 396 into place.
397 397 """
398 398
399 399 if os.path.isdir(dst):
400 400 dst = os.path.join(dst, os.path.basename(src))
401 401
402 402 link_errno = link(src, dst)
403 403 if link_errno == errno.EEXIST:
404 404 if os.stat(src).st_ino == os.stat(dst).st_ino:
405 405 # dst is already a hard link to the correct file, so we don't need
406 406 # to do anything else. If we try to link and rename the file
407 407 # anyway, we get duplicate files - see http://bugs.python.org/issue21876
408 408 return
409 409
410 410 new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
411 411 try:
412 412 link_or_copy(src, new_dst)
413 413 except:
414 414 try:
415 415 os.remove(new_dst)
416 416 except OSError:
417 417 pass
418 418 raise
419 419 os.rename(new_dst, dst)
420 420 elif link_errno != 0:
421 421 # Either link isn't supported, or the filesystem doesn't support
422 422 # linking, or 'src' and 'dst' are on different filesystems.
423 423 shutil.copy(src, dst)
424 424
425 425 def ensure_dir_exists(path, mode=0o755):
426 426 """ensure that a directory exists
427 427
428 428 If it doesn't exist, try to create it and protect against a race condition
429 429 if another process is doing the same.
430 430
431 431 The default permissions are 755, which differ from os.makedirs default of 777.
432 432 """
433 433 if not os.path.exists(path):
434 434 try:
435 435 os.makedirs(path, mode=mode)
436 436 except OSError as e:
437 437 if e.errno != errno.EEXIST:
438 438 raise
439 439 elif not os.path.isdir(path):
440 440 raise IOError("%r exists but is not a directory" % path)
General Comments 0
You need to be logged in to leave comments. Login now