Show More
@@ -1,672 +1,674 b'' | |||
|
1 | 1 | """Various display related classes. |
|
2 | 2 | |
|
3 | 3 | Authors : MinRK, gregcaporaso, dannystaple |
|
4 | 4 | """ |
|
5 | 5 | from html import escape as html_escape |
|
6 | 6 | from os.path import exists, isfile, splitext, abspath, join, isdir |
|
7 | 7 | from os import walk, sep, fsdecode |
|
8 | 8 | |
|
9 | 9 | from IPython.core.display import DisplayObject, TextDisplayObject |
|
10 | 10 | |
|
11 | 11 | from typing import Tuple, Iterable |
|
12 | 12 | |
|
13 | 13 | __all__ = ['Audio', 'IFrame', 'YouTubeVideo', 'VimeoVideo', 'ScribdDocument', |
|
14 | 14 | 'FileLink', 'FileLinks', 'Code'] |
|
15 | 15 | |
|
16 | 16 | |
|
17 | 17 | class Audio(DisplayObject): |
|
18 | 18 | """Create an audio object. |
|
19 | 19 | |
|
20 | 20 | When this object is returned by an input cell or passed to the |
|
21 | 21 | display function, it will result in Audio controls being displayed |
|
22 | 22 | in the frontend (only works in the notebook). |
|
23 | 23 | |
|
24 | 24 | Parameters |
|
25 | 25 | ---------- |
|
26 | 26 | data : numpy array, list, unicode, str or bytes |
|
27 | 27 | Can be one of |
|
28 | 28 | |
|
29 | 29 | * Numpy 1d array containing the desired waveform (mono) |
|
30 | 30 | * Numpy 2d array containing waveforms for each channel. |
|
31 | 31 | Shape=(NCHAN, NSAMPLES). For the standard channel order, see |
|
32 | 32 | http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx |
|
33 | 33 | * List of float or integer representing the waveform (mono) |
|
34 | 34 | * String containing the filename |
|
35 | 35 | * Bytestring containing raw PCM data or |
|
36 | 36 | * URL pointing to a file on the web. |
|
37 | 37 | |
|
38 | 38 | If the array option is used, the waveform will be normalized. |
|
39 | 39 | |
|
40 | 40 | If a filename or url is used, the format support will be browser |
|
41 | 41 | dependent. |
|
42 | 42 | url : unicode |
|
43 | 43 | A URL to download the data from. |
|
44 | 44 | filename : unicode |
|
45 | 45 | Path to a local file to load the data from. |
|
46 | 46 | embed : boolean |
|
47 | 47 | Should the audio data be embedded using a data URI (True) or should |
|
48 | 48 | the original source be referenced. Set this to True if you want the |
|
49 | 49 | audio to playable later with no internet connection in the notebook. |
|
50 | 50 | |
|
51 | 51 | Default is `True`, unless the keyword argument `url` is set, then |
|
52 | 52 | default value is `False`. |
|
53 | 53 | rate : integer |
|
54 | 54 | The sampling rate of the raw data. |
|
55 | 55 | Only required when data parameter is being used as an array |
|
56 | 56 | autoplay : bool |
|
57 | 57 | Set to True if the audio should immediately start playing. |
|
58 | 58 | Default is `False`. |
|
59 | 59 | normalize : bool |
|
60 | 60 | Whether audio should be normalized (rescaled) to the maximum possible |
|
61 | 61 | range. Default is `True`. When set to `False`, `data` must be between |
|
62 | 62 | -1 and 1 (inclusive), otherwise an error is raised. |
|
63 | 63 | Applies only when `data` is a list or array of samples; other types of |
|
64 | 64 | audio are never normalized. |
|
65 | 65 | |
|
66 | 66 | Examples |
|
67 | 67 | -------- |
|
68 | 68 | |
|
69 | 69 | Generate a sound |
|
70 | 70 | |
|
71 | 71 | >>> import numpy as np |
|
72 |
|
|
|
73 |
|
|
|
74 |
|
|
|
75 |
|
|
|
72 | >>> framerate = 44100 | |
|
73 | >>> t = np.linspace(0,5,framerate*5) | |
|
74 | >>> data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t) | |
|
75 | >>> Audio(data, rate=framerate) | |
|
76 | <IPython.lib.display.Audio object> | |
|
76 | 77 | |
|
77 | 78 | Can also do stereo or more channels |
|
78 | 79 | |
|
79 | 80 | >>> dataleft = np.sin(2*np.pi*220*t) |
|
80 |
|
|
|
81 |
|
|
|
81 | >>> dataright = np.sin(2*np.pi*224*t) | |
|
82 | >>> Audio([dataleft, dataright], rate=framerate) | |
|
83 | <IPython.lib.display.Audio object> | |
|
82 | 84 | |
|
83 | 85 | From URL: |
|
84 | 86 | |
|
85 | >>> Audio("http://www.nch.com.au/acm/8k16bitpcm.wav") | |
|
86 | >>> Audio(url="http://www.w3schools.com/html/horse.ogg") | |
|
87 | >>> Audio("http://www.nch.com.au/acm/8k16bitpcm.wav") # doctest: +SKIP | |
|
88 | >>> Audio(url="http://www.w3schools.com/html/horse.ogg") # doctest: +SKIP | |
|
87 | 89 | |
|
88 | 90 | From a File: |
|
89 | 91 | |
|
90 | >>> Audio('/path/to/sound.wav') | |
|
91 | >>> Audio(filename='/path/to/sound.ogg') | |
|
92 | >>> Audio('/path/to/sound.wav') # doctest: +SKIP | |
|
93 | >>> Audio(filename='/path/to/sound.ogg') # doctest: +SKIP | |
|
92 | 94 | |
|
93 | 95 | From Bytes: |
|
94 | 96 | |
|
95 | >>> Audio(b'RAW_WAV_DATA..') | |
|
96 | >>> Audio(data=b'RAW_WAV_DATA..') | |
|
97 | >>> Audio(b'RAW_WAV_DATA..') # doctest: +SKIP | |
|
98 | >>> Audio(data=b'RAW_WAV_DATA..') # doctest: +SKIP | |
|
97 | 99 | |
|
98 | 100 | See Also |
|
99 | 101 | -------- |
|
100 | 102 | ipywidgets.Audio |
|
101 | 103 | |
|
102 | 104 | AUdio widget with more more flexibility and options. |
|
103 | 105 | |
|
104 | 106 | """ |
|
105 | 107 | _read_flags = 'rb' |
|
106 | 108 | |
|
107 | 109 | def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, autoplay=False, normalize=True, *, |
|
108 | 110 | element_id=None): |
|
109 | 111 | if filename is None and url is None and data is None: |
|
110 | 112 | raise ValueError("No audio data found. Expecting filename, url, or data.") |
|
111 | 113 | if embed is False and url is None: |
|
112 | 114 | raise ValueError("No url found. Expecting url when embed=False") |
|
113 | 115 | |
|
114 | 116 | if url is not None and embed is not True: |
|
115 | 117 | self.embed = False |
|
116 | 118 | else: |
|
117 | 119 | self.embed = True |
|
118 | 120 | self.autoplay = autoplay |
|
119 | 121 | self.element_id = element_id |
|
120 | 122 | super(Audio, self).__init__(data=data, url=url, filename=filename) |
|
121 | 123 | |
|
122 | 124 | if self.data is not None and not isinstance(self.data, bytes): |
|
123 | 125 | if rate is None: |
|
124 | 126 | raise ValueError("rate must be specified when data is a numpy array or list of audio samples.") |
|
125 | 127 | self.data = Audio._make_wav(data, rate, normalize) |
|
126 | 128 | |
|
127 | 129 | def reload(self): |
|
128 | 130 | """Reload the raw data from file or URL.""" |
|
129 | 131 | import mimetypes |
|
130 | 132 | if self.embed: |
|
131 | 133 | super(Audio, self).reload() |
|
132 | 134 | |
|
133 | 135 | if self.filename is not None: |
|
134 | 136 | self.mimetype = mimetypes.guess_type(self.filename)[0] |
|
135 | 137 | elif self.url is not None: |
|
136 | 138 | self.mimetype = mimetypes.guess_type(self.url)[0] |
|
137 | 139 | else: |
|
138 | 140 | self.mimetype = "audio/wav" |
|
139 | 141 | |
|
140 | 142 | @staticmethod |
|
141 | 143 | def _make_wav(data, rate, normalize): |
|
142 | 144 | """ Transform a numpy array to a PCM bytestring """ |
|
143 | 145 | from io import BytesIO |
|
144 | 146 | import wave |
|
145 | 147 | |
|
146 | 148 | try: |
|
147 | 149 | scaled, nchan = Audio._validate_and_normalize_with_numpy(data, normalize) |
|
148 | 150 | except ImportError: |
|
149 | 151 | scaled, nchan = Audio._validate_and_normalize_without_numpy(data, normalize) |
|
150 | 152 | |
|
151 | 153 | fp = BytesIO() |
|
152 | 154 | waveobj = wave.open(fp,mode='wb') |
|
153 | 155 | waveobj.setnchannels(nchan) |
|
154 | 156 | waveobj.setframerate(rate) |
|
155 | 157 | waveobj.setsampwidth(2) |
|
156 | 158 | waveobj.setcomptype('NONE','NONE') |
|
157 | 159 | waveobj.writeframes(scaled) |
|
158 | 160 | val = fp.getvalue() |
|
159 | 161 | waveobj.close() |
|
160 | 162 | |
|
161 | 163 | return val |
|
162 | 164 | |
|
163 | 165 | @staticmethod |
|
164 | 166 | def _validate_and_normalize_with_numpy(data, normalize) -> Tuple[bytes, int]: |
|
165 | 167 | import numpy as np |
|
166 | 168 | |
|
167 | 169 | data = np.array(data, dtype=float) |
|
168 | 170 | if len(data.shape) == 1: |
|
169 | 171 | nchan = 1 |
|
170 | 172 | elif len(data.shape) == 2: |
|
171 | 173 | # In wave files,channels are interleaved. E.g., |
|
172 | 174 | # "L1R1L2R2..." for stereo. See |
|
173 | 175 | # http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx |
|
174 | 176 | # for channel ordering |
|
175 | 177 | nchan = data.shape[0] |
|
176 | 178 | data = data.T.ravel() |
|
177 | 179 | else: |
|
178 | 180 | raise ValueError('Array audio input must be a 1D or 2D array') |
|
179 | 181 | |
|
180 | 182 | max_abs_value = np.max(np.abs(data)) |
|
181 | 183 | normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize) |
|
182 | 184 | scaled = data / normalization_factor * 32767 |
|
183 | 185 | return scaled.astype("<h").tobytes(), nchan |
|
184 | 186 | |
|
185 | 187 | @staticmethod |
|
186 | 188 | def _validate_and_normalize_without_numpy(data, normalize): |
|
187 | 189 | import array |
|
188 | 190 | import sys |
|
189 | 191 | |
|
190 | 192 | data = array.array('f', data) |
|
191 | 193 | |
|
192 | 194 | try: |
|
193 | 195 | max_abs_value = float(max([abs(x) for x in data])) |
|
194 | 196 | except TypeError as e: |
|
195 | 197 | raise TypeError('Only lists of mono audio are ' |
|
196 | 198 | 'supported if numpy is not installed') from e |
|
197 | 199 | |
|
198 | 200 | normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize) |
|
199 | 201 | scaled = array.array('h', [int(x / normalization_factor * 32767) for x in data]) |
|
200 | 202 | if sys.byteorder == 'big': |
|
201 | 203 | scaled.byteswap() |
|
202 | 204 | nchan = 1 |
|
203 | 205 | return scaled.tobytes(), nchan |
|
204 | 206 | |
|
205 | 207 | @staticmethod |
|
206 | 208 | def _get_normalization_factor(max_abs_value, normalize): |
|
207 | 209 | if not normalize and max_abs_value > 1: |
|
208 | 210 | raise ValueError('Audio data must be between -1 and 1 when normalize=False.') |
|
209 | 211 | return max_abs_value if normalize else 1 |
|
210 | 212 | |
|
211 | 213 | def _data_and_metadata(self): |
|
212 | 214 | """shortcut for returning metadata with url information, if defined""" |
|
213 | 215 | md = {} |
|
214 | 216 | if self.url: |
|
215 | 217 | md['url'] = self.url |
|
216 | 218 | if md: |
|
217 | 219 | return self.data, md |
|
218 | 220 | else: |
|
219 | 221 | return self.data |
|
220 | 222 | |
|
221 | 223 | def _repr_html_(self): |
|
222 | 224 | src = """ |
|
223 | 225 | <audio {element_id} controls="controls" {autoplay}> |
|
224 | 226 | <source src="{src}" type="{type}" /> |
|
225 | 227 | Your browser does not support the audio element. |
|
226 | 228 | </audio> |
|
227 | 229 | """ |
|
228 | 230 | return src.format(src=self.src_attr(), type=self.mimetype, autoplay=self.autoplay_attr(), |
|
229 | 231 | element_id=self.element_id_attr()) |
|
230 | 232 | |
|
231 | 233 | def src_attr(self): |
|
232 | 234 | import base64 |
|
233 | 235 | if self.embed and (self.data is not None): |
|
234 | 236 | data = base64=base64.b64encode(self.data).decode('ascii') |
|
235 | 237 | return """data:{type};base64,{base64}""".format(type=self.mimetype, |
|
236 | 238 | base64=data) |
|
237 | 239 | elif self.url is not None: |
|
238 | 240 | return self.url |
|
239 | 241 | else: |
|
240 | 242 | return "" |
|
241 | 243 | |
|
242 | 244 | def autoplay_attr(self): |
|
243 | 245 | if(self.autoplay): |
|
244 | 246 | return 'autoplay="autoplay"' |
|
245 | 247 | else: |
|
246 | 248 | return '' |
|
247 | 249 | |
|
248 | 250 | def element_id_attr(self): |
|
249 | 251 | if (self.element_id): |
|
250 | 252 | return 'id="{element_id}"'.format(element_id=self.element_id) |
|
251 | 253 | else: |
|
252 | 254 | return '' |
|
253 | 255 | |
|
254 | 256 | class IFrame(object): |
|
255 | 257 | """ |
|
256 | 258 | Generic class to embed an iframe in an IPython notebook |
|
257 | 259 | """ |
|
258 | 260 | |
|
259 | 261 | iframe = """ |
|
260 | 262 | <iframe |
|
261 | 263 | width="{width}" |
|
262 | 264 | height="{height}" |
|
263 | 265 | src="{src}{params}" |
|
264 | 266 | frameborder="0" |
|
265 | 267 | allowfullscreen |
|
266 | 268 | {extras} |
|
267 | 269 | ></iframe> |
|
268 | 270 | """ |
|
269 | 271 | |
|
270 | 272 | def __init__(self, src, width, height, extras: Iterable[str] = None, **kwargs): |
|
271 | 273 | if extras is None: |
|
272 | 274 | extras = [] |
|
273 | 275 | |
|
274 | 276 | self.src = src |
|
275 | 277 | self.width = width |
|
276 | 278 | self.height = height |
|
277 | 279 | self.extras = extras |
|
278 | 280 | self.params = kwargs |
|
279 | 281 | |
|
280 | 282 | def _repr_html_(self): |
|
281 | 283 | """return the embed iframe""" |
|
282 | 284 | if self.params: |
|
283 | 285 | from urllib.parse import urlencode |
|
284 | 286 | params = "?" + urlencode(self.params) |
|
285 | 287 | else: |
|
286 | 288 | params = "" |
|
287 | 289 | return self.iframe.format( |
|
288 | 290 | src=self.src, |
|
289 | 291 | width=self.width, |
|
290 | 292 | height=self.height, |
|
291 | 293 | params=params, |
|
292 | 294 | extras=" ".join(self.extras), |
|
293 | 295 | ) |
|
294 | 296 | |
|
295 | 297 | |
|
296 | 298 | class YouTubeVideo(IFrame): |
|
297 | 299 | """Class for embedding a YouTube Video in an IPython session, based on its video id. |
|
298 | 300 | |
|
299 | 301 | e.g. to embed the video from https://www.youtube.com/watch?v=foo , you would |
|
300 | 302 | do:: |
|
301 | 303 | |
|
302 | 304 | vid = YouTubeVideo("foo") |
|
303 | 305 | display(vid) |
|
304 | 306 | |
|
305 | 307 | To start from 30 seconds:: |
|
306 | 308 | |
|
307 | 309 | vid = YouTubeVideo("abc", start=30) |
|
308 | 310 | display(vid) |
|
309 | 311 | |
|
310 | 312 | To calculate seconds from time as hours, minutes, seconds use |
|
311 | 313 | :class:`datetime.timedelta`:: |
|
312 | 314 | |
|
313 | 315 | start=int(timedelta(hours=1, minutes=46, seconds=40).total_seconds()) |
|
314 | 316 | |
|
315 | 317 | Other parameters can be provided as documented at |
|
316 | 318 | https://developers.google.com/youtube/player_parameters#Parameters |
|
317 | 319 | |
|
318 | 320 | When converting the notebook using nbconvert, a jpeg representation of the video |
|
319 | 321 | will be inserted in the document. |
|
320 | 322 | """ |
|
321 | 323 | |
|
322 | 324 | def __init__(self, id, width=400, height=300, allow_autoplay=False, **kwargs): |
|
323 | 325 | self.id=id |
|
324 | 326 | src = "https://www.youtube.com/embed/{0}".format(id) |
|
325 | 327 | if allow_autoplay: |
|
326 | 328 | extras = list(kwargs.get("extras", [])) + ['allow="autoplay"'] |
|
327 | 329 | kwargs.update(autoplay=1, extras=extras) |
|
328 | 330 | super(YouTubeVideo, self).__init__(src, width, height, **kwargs) |
|
329 | 331 | |
|
330 | 332 | def _repr_jpeg_(self): |
|
331 | 333 | # Deferred import |
|
332 | 334 | from urllib.request import urlopen |
|
333 | 335 | |
|
334 | 336 | try: |
|
335 | 337 | return urlopen("https://img.youtube.com/vi/{id}/hqdefault.jpg".format(id=self.id)).read() |
|
336 | 338 | except IOError: |
|
337 | 339 | return None |
|
338 | 340 | |
|
339 | 341 | class VimeoVideo(IFrame): |
|
340 | 342 | """ |
|
341 | 343 | Class for embedding a Vimeo video in an IPython session, based on its video id. |
|
342 | 344 | """ |
|
343 | 345 | |
|
344 | 346 | def __init__(self, id, width=400, height=300, **kwargs): |
|
345 | 347 | src="https://player.vimeo.com/video/{0}".format(id) |
|
346 | 348 | super(VimeoVideo, self).__init__(src, width, height, **kwargs) |
|
347 | 349 | |
|
348 | 350 | class ScribdDocument(IFrame): |
|
349 | 351 | """ |
|
350 | 352 | Class for embedding a Scribd document in an IPython session |
|
351 | 353 | |
|
352 | 354 | Use the start_page params to specify a starting point in the document |
|
353 | 355 | Use the view_mode params to specify display type one off scroll | slideshow | book |
|
354 | 356 | |
|
355 | 357 | e.g to Display Wes' foundational paper about PANDAS in book mode from page 3 |
|
356 | 358 | |
|
357 | 359 | ScribdDocument(71048089, width=800, height=400, start_page=3, view_mode="book") |
|
358 | 360 | """ |
|
359 | 361 | |
|
360 | 362 | def __init__(self, id, width=400, height=300, **kwargs): |
|
361 | 363 | src="https://www.scribd.com/embeds/{0}/content".format(id) |
|
362 | 364 | super(ScribdDocument, self).__init__(src, width, height, **kwargs) |
|
363 | 365 | |
|
364 | 366 | class FileLink(object): |
|
365 | 367 | """Class for embedding a local file link in an IPython session, based on path |
|
366 | 368 | |
|
367 | 369 | e.g. to embed a link that was generated in the IPython notebook as my/data.txt |
|
368 | 370 | |
|
369 | 371 | you would do:: |
|
370 | 372 | |
|
371 | 373 | local_file = FileLink("my/data.txt") |
|
372 | 374 | display(local_file) |
|
373 | 375 | |
|
374 | 376 | or in the HTML notebook, just:: |
|
375 | 377 | |
|
376 | 378 | FileLink("my/data.txt") |
|
377 | 379 | """ |
|
378 | 380 | |
|
379 | 381 | html_link_str = "<a href='%s' target='_blank'>%s</a>" |
|
380 | 382 | |
|
381 | 383 | def __init__(self, |
|
382 | 384 | path, |
|
383 | 385 | url_prefix='', |
|
384 | 386 | result_html_prefix='', |
|
385 | 387 | result_html_suffix='<br>'): |
|
386 | 388 | """ |
|
387 | 389 | Parameters |
|
388 | 390 | ---------- |
|
389 | 391 | path : str |
|
390 | 392 | path to the file or directory that should be formatted |
|
391 | 393 | url_prefix : str |
|
392 | 394 | prefix to be prepended to all files to form a working link [default: |
|
393 | 395 | ''] |
|
394 | 396 | result_html_prefix : str |
|
395 | 397 | text to append to beginning to link [default: ''] |
|
396 | 398 | result_html_suffix : str |
|
397 | 399 | text to append at the end of link [default: '<br>'] |
|
398 | 400 | """ |
|
399 | 401 | if isdir(path): |
|
400 | 402 | raise ValueError("Cannot display a directory using FileLink. " |
|
401 | 403 | "Use FileLinks to display '%s'." % path) |
|
402 | 404 | self.path = fsdecode(path) |
|
403 | 405 | self.url_prefix = url_prefix |
|
404 | 406 | self.result_html_prefix = result_html_prefix |
|
405 | 407 | self.result_html_suffix = result_html_suffix |
|
406 | 408 | |
|
407 | 409 | def _format_path(self): |
|
408 | 410 | fp = ''.join([self.url_prefix, html_escape(self.path)]) |
|
409 | 411 | return ''.join([self.result_html_prefix, |
|
410 | 412 | self.html_link_str % \ |
|
411 | 413 | (fp, html_escape(self.path, quote=False)), |
|
412 | 414 | self.result_html_suffix]) |
|
413 | 415 | |
|
414 | 416 | def _repr_html_(self): |
|
415 | 417 | """return html link to file |
|
416 | 418 | """ |
|
417 | 419 | if not exists(self.path): |
|
418 | 420 | return ("Path (<tt>%s</tt>) doesn't exist. " |
|
419 | 421 | "It may still be in the process of " |
|
420 | 422 | "being generated, or you may have the " |
|
421 | 423 | "incorrect path." % self.path) |
|
422 | 424 | |
|
423 | 425 | return self._format_path() |
|
424 | 426 | |
|
425 | 427 | def __repr__(self): |
|
426 | 428 | """return absolute path to file |
|
427 | 429 | """ |
|
428 | 430 | return abspath(self.path) |
|
429 | 431 | |
|
430 | 432 | class FileLinks(FileLink): |
|
431 | 433 | """Class for embedding local file links in an IPython session, based on path |
|
432 | 434 | |
|
433 | 435 | e.g. to embed links to files that were generated in the IPython notebook |
|
434 | 436 | under ``my/data``, you would do:: |
|
435 | 437 | |
|
436 | 438 | local_files = FileLinks("my/data") |
|
437 | 439 | display(local_files) |
|
438 | 440 | |
|
439 | 441 | or in the HTML notebook, just:: |
|
440 | 442 | |
|
441 | 443 | FileLinks("my/data") |
|
442 | 444 | """ |
|
443 | 445 | def __init__(self, |
|
444 | 446 | path, |
|
445 | 447 | url_prefix='', |
|
446 | 448 | included_suffixes=None, |
|
447 | 449 | result_html_prefix='', |
|
448 | 450 | result_html_suffix='<br>', |
|
449 | 451 | notebook_display_formatter=None, |
|
450 | 452 | terminal_display_formatter=None, |
|
451 | 453 | recursive=True): |
|
452 | 454 | """ |
|
453 | 455 | See :class:`FileLink` for the ``path``, ``url_prefix``, |
|
454 | 456 | ``result_html_prefix`` and ``result_html_suffix`` parameters. |
|
455 | 457 | |
|
456 | 458 | included_suffixes : list |
|
457 | 459 | Filename suffixes to include when formatting output [default: include |
|
458 | 460 | all files] |
|
459 | 461 | |
|
460 | 462 | notebook_display_formatter : function |
|
461 | 463 | Used to format links for display in the notebook. See discussion of |
|
462 | 464 | formatter functions below. |
|
463 | 465 | |
|
464 | 466 | terminal_display_formatter : function |
|
465 | 467 | Used to format links for display in the terminal. See discussion of |
|
466 | 468 | formatter functions below. |
|
467 | 469 | |
|
468 | 470 | Formatter functions must be of the form:: |
|
469 | 471 | |
|
470 | 472 | f(dirname, fnames, included_suffixes) |
|
471 | 473 | |
|
472 | 474 | dirname : str |
|
473 | 475 | The name of a directory |
|
474 | 476 | fnames : list |
|
475 | 477 | The files in that directory |
|
476 | 478 | included_suffixes : list |
|
477 | 479 | The file suffixes that should be included in the output (passing None |
|
478 | 480 | meansto include all suffixes in the output in the built-in formatters) |
|
479 | 481 | recursive : boolean |
|
480 | 482 | Whether to recurse into subdirectories. Default is True. |
|
481 | 483 | |
|
482 | 484 | The function should return a list of lines that will be printed in the |
|
483 | 485 | notebook (if passing notebook_display_formatter) or the terminal (if |
|
484 | 486 | passing terminal_display_formatter). This function is iterated over for |
|
485 | 487 | each directory in self.path. Default formatters are in place, can be |
|
486 | 488 | passed here to support alternative formatting. |
|
487 | 489 | |
|
488 | 490 | """ |
|
489 | 491 | if isfile(path): |
|
490 | 492 | raise ValueError("Cannot display a file using FileLinks. " |
|
491 | 493 | "Use FileLink to display '%s'." % path) |
|
492 | 494 | self.included_suffixes = included_suffixes |
|
493 | 495 | # remove trailing slashes for more consistent output formatting |
|
494 | 496 | path = path.rstrip('/') |
|
495 | 497 | |
|
496 | 498 | self.path = path |
|
497 | 499 | self.url_prefix = url_prefix |
|
498 | 500 | self.result_html_prefix = result_html_prefix |
|
499 | 501 | self.result_html_suffix = result_html_suffix |
|
500 | 502 | |
|
501 | 503 | self.notebook_display_formatter = \ |
|
502 | 504 | notebook_display_formatter or self._get_notebook_display_formatter() |
|
503 | 505 | self.terminal_display_formatter = \ |
|
504 | 506 | terminal_display_formatter or self._get_terminal_display_formatter() |
|
505 | 507 | |
|
506 | 508 | self.recursive = recursive |
|
507 | 509 | |
|
508 | 510 | def _get_display_formatter(self, |
|
509 | 511 | dirname_output_format, |
|
510 | 512 | fname_output_format, |
|
511 | 513 | fp_format, |
|
512 | 514 | fp_cleaner=None): |
|
513 | 515 | """ generate built-in formatter function |
|
514 | 516 | |
|
515 | 517 | this is used to define both the notebook and terminal built-in |
|
516 | 518 | formatters as they only differ by some wrapper text for each entry |
|
517 | 519 | |
|
518 | 520 | dirname_output_format: string to use for formatting directory |
|
519 | 521 | names, dirname will be substituted for a single "%s" which |
|
520 | 522 | must appear in this string |
|
521 | 523 | fname_output_format: string to use for formatting file names, |
|
522 | 524 | if a single "%s" appears in the string, fname will be substituted |
|
523 | 525 | if two "%s" appear in the string, the path to fname will be |
|
524 | 526 | substituted for the first and fname will be substituted for the |
|
525 | 527 | second |
|
526 | 528 | fp_format: string to use for formatting filepaths, must contain |
|
527 | 529 | exactly two "%s" and the dirname will be substituted for the first |
|
528 | 530 | and fname will be substituted for the second |
|
529 | 531 | """ |
|
530 | 532 | def f(dirname, fnames, included_suffixes=None): |
|
531 | 533 | result = [] |
|
532 | 534 | # begin by figuring out which filenames, if any, |
|
533 | 535 | # are going to be displayed |
|
534 | 536 | display_fnames = [] |
|
535 | 537 | for fname in fnames: |
|
536 | 538 | if (isfile(join(dirname,fname)) and |
|
537 | 539 | (included_suffixes is None or |
|
538 | 540 | splitext(fname)[1] in included_suffixes)): |
|
539 | 541 | display_fnames.append(fname) |
|
540 | 542 | |
|
541 | 543 | if len(display_fnames) == 0: |
|
542 | 544 | # if there are no filenames to display, don't print anything |
|
543 | 545 | # (not even the directory name) |
|
544 | 546 | pass |
|
545 | 547 | else: |
|
546 | 548 | # otherwise print the formatted directory name followed by |
|
547 | 549 | # the formatted filenames |
|
548 | 550 | dirname_output_line = dirname_output_format % dirname |
|
549 | 551 | result.append(dirname_output_line) |
|
550 | 552 | for fname in display_fnames: |
|
551 | 553 | fp = fp_format % (dirname,fname) |
|
552 | 554 | if fp_cleaner is not None: |
|
553 | 555 | fp = fp_cleaner(fp) |
|
554 | 556 | try: |
|
555 | 557 | # output can include both a filepath and a filename... |
|
556 | 558 | fname_output_line = fname_output_format % (fp, fname) |
|
557 | 559 | except TypeError: |
|
558 | 560 | # ... or just a single filepath |
|
559 | 561 | fname_output_line = fname_output_format % fname |
|
560 | 562 | result.append(fname_output_line) |
|
561 | 563 | return result |
|
562 | 564 | return f |
|
563 | 565 | |
|
564 | 566 | def _get_notebook_display_formatter(self, |
|
565 | 567 | spacer=" "): |
|
566 | 568 | """ generate function to use for notebook formatting |
|
567 | 569 | """ |
|
568 | 570 | dirname_output_format = \ |
|
569 | 571 | self.result_html_prefix + "%s/" + self.result_html_suffix |
|
570 | 572 | fname_output_format = \ |
|
571 | 573 | self.result_html_prefix + spacer + self.html_link_str + self.result_html_suffix |
|
572 | 574 | fp_format = self.url_prefix + '%s/%s' |
|
573 | 575 | if sep == "\\": |
|
574 | 576 | # Working on a platform where the path separator is "\", so |
|
575 | 577 | # must convert these to "/" for generating a URI |
|
576 | 578 | def fp_cleaner(fp): |
|
577 | 579 | # Replace all occurrences of backslash ("\") with a forward |
|
578 | 580 | # slash ("/") - this is necessary on windows when a path is |
|
579 | 581 | # provided as input, but we must link to a URI |
|
580 | 582 | return fp.replace('\\','/') |
|
581 | 583 | else: |
|
582 | 584 | fp_cleaner = None |
|
583 | 585 | |
|
584 | 586 | return self._get_display_formatter(dirname_output_format, |
|
585 | 587 | fname_output_format, |
|
586 | 588 | fp_format, |
|
587 | 589 | fp_cleaner) |
|
588 | 590 | |
|
589 | 591 | def _get_terminal_display_formatter(self, |
|
590 | 592 | spacer=" "): |
|
591 | 593 | """ generate function to use for terminal formatting |
|
592 | 594 | """ |
|
593 | 595 | dirname_output_format = "%s/" |
|
594 | 596 | fname_output_format = spacer + "%s" |
|
595 | 597 | fp_format = '%s/%s' |
|
596 | 598 | |
|
597 | 599 | return self._get_display_formatter(dirname_output_format, |
|
598 | 600 | fname_output_format, |
|
599 | 601 | fp_format) |
|
600 | 602 | |
|
601 | 603 | def _format_path(self): |
|
602 | 604 | result_lines = [] |
|
603 | 605 | if self.recursive: |
|
604 | 606 | walked_dir = list(walk(self.path)) |
|
605 | 607 | else: |
|
606 | 608 | walked_dir = [next(walk(self.path))] |
|
607 | 609 | walked_dir.sort() |
|
608 | 610 | for dirname, subdirs, fnames in walked_dir: |
|
609 | 611 | result_lines += self.notebook_display_formatter(dirname, fnames, self.included_suffixes) |
|
610 | 612 | return '\n'.join(result_lines) |
|
611 | 613 | |
|
612 | 614 | def __repr__(self): |
|
613 | 615 | """return newline-separated absolute paths |
|
614 | 616 | """ |
|
615 | 617 | result_lines = [] |
|
616 | 618 | if self.recursive: |
|
617 | 619 | walked_dir = list(walk(self.path)) |
|
618 | 620 | else: |
|
619 | 621 | walked_dir = [next(walk(self.path))] |
|
620 | 622 | walked_dir.sort() |
|
621 | 623 | for dirname, subdirs, fnames in walked_dir: |
|
622 | 624 | result_lines += self.terminal_display_formatter(dirname, fnames, self.included_suffixes) |
|
623 | 625 | return '\n'.join(result_lines) |
|
624 | 626 | |
|
625 | 627 | |
|
626 | 628 | class Code(TextDisplayObject): |
|
627 | 629 | """Display syntax-highlighted source code. |
|
628 | 630 | |
|
629 | 631 | This uses Pygments to highlight the code for HTML and Latex output. |
|
630 | 632 | |
|
631 | 633 | Parameters |
|
632 | 634 | ---------- |
|
633 | 635 | data : str |
|
634 | 636 | The code as a string |
|
635 | 637 | url : str |
|
636 | 638 | A URL to fetch the code from |
|
637 | 639 | filename : str |
|
638 | 640 | A local filename to load the code from |
|
639 | 641 | language : str |
|
640 | 642 | The short name of a Pygments lexer to use for highlighting. |
|
641 | 643 | If not specified, it will guess the lexer based on the filename |
|
642 | 644 | or the code. Available lexers: http://pygments.org/docs/lexers/ |
|
643 | 645 | """ |
|
644 | 646 | def __init__(self, data=None, url=None, filename=None, language=None): |
|
645 | 647 | self.language = language |
|
646 | 648 | super().__init__(data=data, url=url, filename=filename) |
|
647 | 649 | |
|
648 | 650 | def _get_lexer(self): |
|
649 | 651 | if self.language: |
|
650 | 652 | from pygments.lexers import get_lexer_by_name |
|
651 | 653 | return get_lexer_by_name(self.language) |
|
652 | 654 | elif self.filename: |
|
653 | 655 | from pygments.lexers import get_lexer_for_filename |
|
654 | 656 | return get_lexer_for_filename(self.filename) |
|
655 | 657 | else: |
|
656 | 658 | from pygments.lexers import guess_lexer |
|
657 | 659 | return guess_lexer(self.data) |
|
658 | 660 | |
|
659 | 661 | def __repr__(self): |
|
660 | 662 | return self.data |
|
661 | 663 | |
|
662 | 664 | def _repr_html_(self): |
|
663 | 665 | from pygments import highlight |
|
664 | 666 | from pygments.formatters import HtmlFormatter |
|
665 | 667 | fmt = HtmlFormatter() |
|
666 | 668 | style = '<style>{}</style>'.format(fmt.get_style_defs('.output_html')) |
|
667 | 669 | return style + highlight(self.data, self._get_lexer(), fmt) |
|
668 | 670 | |
|
669 | 671 | def _repr_latex_(self): |
|
670 | 672 | from pygments import highlight |
|
671 | 673 | from pygments.formatters import LatexFormatter |
|
672 | 674 | return highlight(self.data, self._get_lexer(), LatexFormatter()) |
@@ -1,532 +1,526 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """ |
|
3 | 3 | Defines a variety of Pygments lexers for highlighting IPython code. |
|
4 | 4 | |
|
5 | 5 | This includes: |
|
6 | 6 | |
|
7 | 7 | IPythonLexer, IPython3Lexer |
|
8 | 8 | Lexers for pure IPython (python + magic/shell commands) |
|
9 | 9 | |
|
10 | 10 | IPythonPartialTracebackLexer, IPythonTracebackLexer |
|
11 | 11 | Supports 2.x and 3.x via keyword `python3`. The partial traceback |
|
12 | 12 | lexer reads everything but the Python code appearing in a traceback. |
|
13 | 13 | The full lexer combines the partial lexer with an IPython lexer. |
|
14 | 14 | |
|
15 | 15 | IPythonConsoleLexer |
|
16 | 16 | A lexer for IPython console sessions, with support for tracebacks. |
|
17 | 17 | |
|
18 | 18 | IPyLexer |
|
19 | 19 | A friendly lexer which examines the first line of text and from it, |
|
20 | 20 | decides whether to use an IPython lexer or an IPython console lexer. |
|
21 | 21 | This is probably the only lexer that needs to be explicitly added |
|
22 | 22 | to Pygments. |
|
23 | 23 | |
|
24 | 24 | """ |
|
25 | 25 | #----------------------------------------------------------------------------- |
|
26 | 26 | # Copyright (c) 2013, the IPython Development Team. |
|
27 | 27 | # |
|
28 | 28 | # Distributed under the terms of the Modified BSD License. |
|
29 | 29 | # |
|
30 | 30 | # The full license is in the file COPYING.txt, distributed with this software. |
|
31 | 31 | #----------------------------------------------------------------------------- |
|
32 | 32 | |
|
33 | 33 | # Standard library |
|
34 | 34 | import re |
|
35 | 35 | |
|
36 | 36 | # Third party |
|
37 | 37 | from pygments.lexers import ( |
|
38 | 38 | BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, PythonLexer, |
|
39 | 39 | Python3Lexer, TexLexer) |
|
40 | 40 | from pygments.lexer import ( |
|
41 | 41 | Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using, |
|
42 | 42 | ) |
|
43 | 43 | from pygments.token import ( |
|
44 | 44 | Generic, Keyword, Literal, Name, Operator, Other, Text, Error, |
|
45 | 45 | ) |
|
46 | 46 | from pygments.util import get_bool_opt |
|
47 | 47 | |
|
48 | 48 | # Local |
|
49 | 49 | |
|
50 | 50 | line_re = re.compile('.*?\n') |
|
51 | 51 | |
|
52 | 52 | __all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer', |
|
53 | 53 | 'IPythonPartialTracebackLexer', 'IPythonTracebackLexer', |
|
54 | 54 | 'IPythonConsoleLexer', 'IPyLexer'] |
|
55 | 55 | |
|
56 | 56 | |
|
57 | 57 | def build_ipy_lexer(python3): |
|
58 | 58 | """Builds IPython lexers depending on the value of `python3`. |
|
59 | 59 | |
|
60 | 60 | The lexer inherits from an appropriate Python lexer and then adds |
|
61 | 61 | information about IPython specific keywords (i.e. magic commands, |
|
62 | 62 | shell commands, etc.) |
|
63 | 63 | |
|
64 | 64 | Parameters |
|
65 | 65 | ---------- |
|
66 | 66 | python3 : bool |
|
67 | 67 | If `True`, then build an IPython lexer from a Python 3 lexer. |
|
68 | 68 | |
|
69 | 69 | """ |
|
70 | 70 | # It would be nice to have a single IPython lexer class which takes |
|
71 | 71 | # a boolean `python3`. But since there are two Python lexer classes, |
|
72 | 72 | # we will also have two IPython lexer classes. |
|
73 | 73 | if python3: |
|
74 | 74 | PyLexer = Python3Lexer |
|
75 | 75 | name = 'IPython3' |
|
76 | 76 | aliases = ['ipython3'] |
|
77 | 77 | doc = """IPython3 Lexer""" |
|
78 | 78 | else: |
|
79 | 79 | PyLexer = PythonLexer |
|
80 | 80 | name = 'IPython' |
|
81 | 81 | aliases = ['ipython2', 'ipython'] |
|
82 | 82 | doc = """IPython Lexer""" |
|
83 | 83 | |
|
84 | 84 | ipython_tokens = [ |
|
85 | 85 | (r'(?s)(\s*)(%%capture)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), |
|
86 | 86 | (r'(?s)(\s*)(%%debug)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), |
|
87 | 87 | (r'(?is)(\s*)(%%html)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(HtmlLexer))), |
|
88 | 88 | (r'(?s)(\s*)(%%javascript)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))), |
|
89 | 89 | (r'(?s)(\s*)(%%js)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))), |
|
90 | 90 | (r'(?s)(\s*)(%%latex)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(TexLexer))), |
|
91 | 91 | (r'(?s)(\s*)(%%perl)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PerlLexer))), |
|
92 | 92 | (r'(?s)(\s*)(%%prun)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), |
|
93 | 93 | (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), |
|
94 | 94 | (r'(?s)(\s*)(%%python)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), |
|
95 | 95 | (r'(?s)(\s*)(%%python2)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PythonLexer))), |
|
96 | 96 | (r'(?s)(\s*)(%%python3)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), |
|
97 | 97 | (r'(?s)(\s*)(%%ruby)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(RubyLexer))), |
|
98 | 98 | (r'(?s)(\s*)(%%time)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), |
|
99 | 99 | (r'(?s)(\s*)(%%timeit)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), |
|
100 | 100 | (r'(?s)(\s*)(%%writefile)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), |
|
101 | 101 | (r'(?s)(\s*)(%%file)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))), |
|
102 | 102 | (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)), |
|
103 | 103 | (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))), |
|
104 | 104 | (r"(%%?)(\w+)(\?\??)$", bygroups(Operator, Keyword, Operator)), |
|
105 | 105 | (r"\b(\?\??)(\s*)$", bygroups(Operator, Text)), |
|
106 | 106 | (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword, |
|
107 | 107 | using(BashLexer), Text)), |
|
108 | 108 | (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)), |
|
109 | 109 | (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)), |
|
110 | 110 | (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)), |
|
111 | 111 | (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)), |
|
112 | 112 | (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)), |
|
113 | 113 | ] |
|
114 | 114 | |
|
115 | 115 | tokens = PyLexer.tokens.copy() |
|
116 | 116 | tokens['root'] = ipython_tokens + tokens['root'] |
|
117 | 117 | |
|
118 | 118 | attrs = {'name': name, 'aliases': aliases, 'filenames': [], |
|
119 | 119 | '__doc__': doc, 'tokens': tokens} |
|
120 | 120 | |
|
121 | 121 | return type(name, (PyLexer,), attrs) |
|
122 | 122 | |
|
123 | 123 | |
|
124 | 124 | IPython3Lexer = build_ipy_lexer(python3=True) |
|
125 | 125 | IPythonLexer = build_ipy_lexer(python3=False) |
|
126 | 126 | |
|
127 | 127 | |
|
128 | 128 | class IPythonPartialTracebackLexer(RegexLexer): |
|
129 | 129 | """ |
|
130 | 130 | Partial lexer for IPython tracebacks. |
|
131 | 131 | |
|
132 | 132 | Handles all the non-python output. |
|
133 | 133 | |
|
134 | 134 | """ |
|
135 | 135 | name = 'IPython Partial Traceback' |
|
136 | 136 | |
|
137 | 137 | tokens = { |
|
138 | 138 | 'root': [ |
|
139 | 139 | # Tracebacks for syntax errors have a different style. |
|
140 | 140 | # For both types of tracebacks, we mark the first line with |
|
141 | 141 | # Generic.Traceback. For syntax errors, we mark the filename |
|
142 | 142 | # as we mark the filenames for non-syntax tracebacks. |
|
143 | 143 | # |
|
144 | 144 | # These two regexps define how IPythonConsoleLexer finds a |
|
145 | 145 | # traceback. |
|
146 | 146 | # |
|
147 | 147 | ## Non-syntax traceback |
|
148 | 148 | (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)), |
|
149 | 149 | ## Syntax traceback |
|
150 | 150 | (r'^( File)(.*)(, line )(\d+\n)', |
|
151 | 151 | bygroups(Generic.Traceback, Name.Namespace, |
|
152 | 152 | Generic.Traceback, Literal.Number.Integer)), |
|
153 | 153 | |
|
154 | 154 | # (Exception Identifier)(Whitespace)(Traceback Message) |
|
155 | 155 | (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)', |
|
156 | 156 | bygroups(Name.Exception, Generic.Whitespace, Text)), |
|
157 | 157 | # (Module/Filename)(Text)(Callee)(Function Signature) |
|
158 | 158 | # Better options for callee and function signature? |
|
159 | 159 | (r'(.*)( in )(.*)(\(.*\)\n)', |
|
160 | 160 | bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)), |
|
161 | 161 | # Regular line: (Whitespace)(Line Number)(Python Code) |
|
162 | 162 | (r'(\s*?)(\d+)(.*?\n)', |
|
163 | 163 | bygroups(Generic.Whitespace, Literal.Number.Integer, Other)), |
|
164 | 164 | # Emphasized line: (Arrow)(Line Number)(Python Code) |
|
165 | 165 | # Using Exception token so arrow color matches the Exception. |
|
166 | 166 | (r'(-*>?\s?)(\d+)(.*?\n)', |
|
167 | 167 | bygroups(Name.Exception, Literal.Number.Integer, Other)), |
|
168 | 168 | # (Exception Identifier)(Message) |
|
169 | 169 | (r'(?u)(^[^\d\W]\w*)(:.*?\n)', |
|
170 | 170 | bygroups(Name.Exception, Text)), |
|
171 | 171 | # Tag everything else as Other, will be handled later. |
|
172 | 172 | (r'.*\n', Other), |
|
173 | 173 | ], |
|
174 | 174 | } |
|
175 | 175 | |
|
176 | 176 | |
|
177 | 177 | class IPythonTracebackLexer(DelegatingLexer): |
|
178 | 178 | """ |
|
179 | 179 | IPython traceback lexer. |
|
180 | 180 | |
|
181 | 181 | For doctests, the tracebacks can be snipped as much as desired with the |
|
182 | 182 | exception to the lines that designate a traceback. For non-syntax error |
|
183 | 183 | tracebacks, this is the line of hyphens. For syntax error tracebacks, |
|
184 | 184 | this is the line which lists the File and line number. |
|
185 | 185 | |
|
186 | 186 | """ |
|
187 | 187 | # The lexer inherits from DelegatingLexer. The "root" lexer is an |
|
188 | 188 | # appropriate IPython lexer, which depends on the value of the boolean |
|
189 | 189 | # `python3`. First, we parse with the partial IPython traceback lexer. |
|
190 | 190 | # Then, any code marked with the "Other" token is delegated to the root |
|
191 | 191 | # lexer. |
|
192 | 192 | # |
|
193 | 193 | name = 'IPython Traceback' |
|
194 | 194 | aliases = ['ipythontb'] |
|
195 | 195 | |
|
196 | 196 | def __init__(self, **options): |
|
197 | 197 | self.python3 = get_bool_opt(options, 'python3', False) |
|
198 | 198 | if self.python3: |
|
199 | 199 | self.aliases = ['ipython3tb'] |
|
200 | 200 | else: |
|
201 | 201 | self.aliases = ['ipython2tb', 'ipythontb'] |
|
202 | 202 | |
|
203 | 203 | if self.python3: |
|
204 | 204 | IPyLexer = IPython3Lexer |
|
205 | 205 | else: |
|
206 | 206 | IPyLexer = IPythonLexer |
|
207 | 207 | |
|
208 | 208 | DelegatingLexer.__init__(self, IPyLexer, |
|
209 | 209 | IPythonPartialTracebackLexer, **options) |
|
210 | 210 | |
|
211 | 211 | class IPythonConsoleLexer(Lexer): |
|
212 | 212 | """ |
|
213 | 213 | An IPython console lexer for IPython code-blocks and doctests, such as: |
|
214 | 214 | |
|
215 | 215 | .. code-block:: rst |
|
216 | 216 | |
|
217 | 217 | .. code-block:: ipythonconsole |
|
218 | 218 | |
|
219 | 219 | In [1]: a = 'foo' |
|
220 | 220 | |
|
221 | 221 | In [2]: a |
|
222 | 222 | Out[2]: 'foo' |
|
223 | 223 | |
|
224 |
In [3]: print |
|
|
224 | In [3]: print(a) | |
|
225 | 225 | foo |
|
226 | 226 | |
|
227 | In [4]: 1 / 0 | |
|
228 | ||
|
229 | 227 | |
|
230 | 228 | Support is also provided for IPython exceptions: |
|
231 | 229 | |
|
232 | 230 | .. code-block:: rst |
|
233 | 231 | |
|
234 | 232 | .. code-block:: ipythonconsole |
|
235 | 233 | |
|
236 | 234 | In [1]: raise Exception |
|
237 | ||
|
238 | --------------------------------------------------------------------------- | |
|
239 | Exception Traceback (most recent call last) | |
|
240 | <ipython-input-1-fca2ab0ca76b> in <module> | |
|
241 | ----> 1 raise Exception | |
|
242 | ||
|
243 | Exception: | |
|
235 | Traceback (most recent call last): | |
|
236 | ... | |
|
237 | Exception | |
|
244 | 238 | |
|
245 | 239 | """ |
|
246 | 240 | name = 'IPython console session' |
|
247 | 241 | aliases = ['ipythonconsole'] |
|
248 | 242 | mimetypes = ['text/x-ipython-console'] |
|
249 | 243 | |
|
250 | 244 | # The regexps used to determine what is input and what is output. |
|
251 | 245 | # The default prompts for IPython are: |
|
252 | 246 | # |
|
253 | 247 | # in = 'In [#]: ' |
|
254 | 248 | # continuation = ' .D.: ' |
|
255 | 249 | # template = 'Out[#]: ' |
|
256 | 250 | # |
|
257 | 251 | # Where '#' is the 'prompt number' or 'execution count' and 'D' |
|
258 | 252 | # D is a number of dots matching the width of the execution count |
|
259 | 253 | # |
|
260 | 254 | in1_regex = r'In \[[0-9]+\]: ' |
|
261 | 255 | in2_regex = r' \.\.+\.: ' |
|
262 | 256 | out_regex = r'Out\[[0-9]+\]: ' |
|
263 | 257 | |
|
264 | 258 | #: The regex to determine when a traceback starts. |
|
265 | 259 | ipytb_start = re.compile(r'^(\^C)?(-+\n)|^( File)(.*)(, line )(\d+\n)') |
|
266 | 260 | |
|
267 | 261 | def __init__(self, **options): |
|
268 | 262 | """Initialize the IPython console lexer. |
|
269 | 263 | |
|
270 | 264 | Parameters |
|
271 | 265 | ---------- |
|
272 | 266 | python3 : bool |
|
273 | 267 | If `True`, then the console inputs are parsed using a Python 3 |
|
274 | 268 | lexer. Otherwise, they are parsed using a Python 2 lexer. |
|
275 | 269 | in1_regex : RegexObject |
|
276 | 270 | The compiled regular expression used to detect the start |
|
277 | 271 | of inputs. Although the IPython configuration setting may have a |
|
278 | 272 | trailing whitespace, do not include it in the regex. If `None`, |
|
279 | 273 | then the default input prompt is assumed. |
|
280 | 274 | in2_regex : RegexObject |
|
281 | 275 | The compiled regular expression used to detect the continuation |
|
282 | 276 | of inputs. Although the IPython configuration setting may have a |
|
283 | 277 | trailing whitespace, do not include it in the regex. If `None`, |
|
284 | 278 | then the default input prompt is assumed. |
|
285 | 279 | out_regex : RegexObject |
|
286 | 280 | The compiled regular expression used to detect outputs. If `None`, |
|
287 | 281 | then the default output prompt is assumed. |
|
288 | 282 | |
|
289 | 283 | """ |
|
290 | 284 | self.python3 = get_bool_opt(options, 'python3', False) |
|
291 | 285 | if self.python3: |
|
292 | 286 | self.aliases = ['ipython3console'] |
|
293 | 287 | else: |
|
294 | 288 | self.aliases = ['ipython2console', 'ipythonconsole'] |
|
295 | 289 | |
|
296 | 290 | in1_regex = options.get('in1_regex', self.in1_regex) |
|
297 | 291 | in2_regex = options.get('in2_regex', self.in2_regex) |
|
298 | 292 | out_regex = options.get('out_regex', self.out_regex) |
|
299 | 293 | |
|
300 | 294 | # So that we can work with input and output prompts which have been |
|
301 | 295 | # rstrip'd (possibly by editors) we also need rstrip'd variants. If |
|
302 | 296 | # we do not do this, then such prompts will be tagged as 'output'. |
|
303 | 297 | # The reason can't just use the rstrip'd variants instead is because |
|
304 | 298 | # we want any whitespace associated with the prompt to be inserted |
|
305 | 299 | # with the token. This allows formatted code to be modified so as hide |
|
306 | 300 | # the appearance of prompts, with the whitespace included. One example |
|
307 | 301 | # use of this is in copybutton.js from the standard lib Python docs. |
|
308 | 302 | in1_regex_rstrip = in1_regex.rstrip() + '\n' |
|
309 | 303 | in2_regex_rstrip = in2_regex.rstrip() + '\n' |
|
310 | 304 | out_regex_rstrip = out_regex.rstrip() + '\n' |
|
311 | 305 | |
|
312 | 306 | # Compile and save them all. |
|
313 | 307 | attrs = ['in1_regex', 'in2_regex', 'out_regex', |
|
314 | 308 | 'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip'] |
|
315 | 309 | for attr in attrs: |
|
316 | 310 | self.__setattr__(attr, re.compile(locals()[attr])) |
|
317 | 311 | |
|
318 | 312 | Lexer.__init__(self, **options) |
|
319 | 313 | |
|
320 | 314 | if self.python3: |
|
321 | 315 | pylexer = IPython3Lexer |
|
322 | 316 | tblexer = IPythonTracebackLexer |
|
323 | 317 | else: |
|
324 | 318 | pylexer = IPythonLexer |
|
325 | 319 | tblexer = IPythonTracebackLexer |
|
326 | 320 | |
|
327 | 321 | self.pylexer = pylexer(**options) |
|
328 | 322 | self.tblexer = tblexer(**options) |
|
329 | 323 | |
|
330 | 324 | self.reset() |
|
331 | 325 | |
|
332 | 326 | def reset(self): |
|
333 | 327 | self.mode = 'output' |
|
334 | 328 | self.index = 0 |
|
335 | 329 | self.buffer = u'' |
|
336 | 330 | self.insertions = [] |
|
337 | 331 | |
|
338 | 332 | def buffered_tokens(self): |
|
339 | 333 | """ |
|
340 | 334 | Generator of unprocessed tokens after doing insertions and before |
|
341 | 335 | changing to a new state. |
|
342 | 336 | |
|
343 | 337 | """ |
|
344 | 338 | if self.mode == 'output': |
|
345 | 339 | tokens = [(0, Generic.Output, self.buffer)] |
|
346 | 340 | elif self.mode == 'input': |
|
347 | 341 | tokens = self.pylexer.get_tokens_unprocessed(self.buffer) |
|
348 | 342 | else: # traceback |
|
349 | 343 | tokens = self.tblexer.get_tokens_unprocessed(self.buffer) |
|
350 | 344 | |
|
351 | 345 | for i, t, v in do_insertions(self.insertions, tokens): |
|
352 | 346 | # All token indexes are relative to the buffer. |
|
353 | 347 | yield self.index + i, t, v |
|
354 | 348 | |
|
355 | 349 | # Clear it all |
|
356 | 350 | self.index += len(self.buffer) |
|
357 | 351 | self.buffer = u'' |
|
358 | 352 | self.insertions = [] |
|
359 | 353 | |
|
360 | 354 | def get_mci(self, line): |
|
361 | 355 | """ |
|
362 | 356 | Parses the line and returns a 3-tuple: (mode, code, insertion). |
|
363 | 357 | |
|
364 | 358 | `mode` is the next mode (or state) of the lexer, and is always equal |
|
365 | 359 | to 'input', 'output', or 'tb'. |
|
366 | 360 | |
|
367 | 361 | `code` is a portion of the line that should be added to the buffer |
|
368 | 362 | corresponding to the next mode and eventually lexed by another lexer. |
|
369 | 363 | For example, `code` could be Python code if `mode` were 'input'. |
|
370 | 364 | |
|
371 | 365 | `insertion` is a 3-tuple (index, token, text) representing an |
|
372 | 366 | unprocessed "token" that will be inserted into the stream of tokens |
|
373 | 367 | that are created from the buffer once we change modes. This is usually |
|
374 | 368 | the input or output prompt. |
|
375 | 369 | |
|
376 | 370 | In general, the next mode depends on current mode and on the contents |
|
377 | 371 | of `line`. |
|
378 | 372 | |
|
379 | 373 | """ |
|
380 | 374 | # To reduce the number of regex match checks, we have multiple |
|
381 | 375 | # 'if' blocks instead of 'if-elif' blocks. |
|
382 | 376 | |
|
383 | 377 | # Check for possible end of input |
|
384 | 378 | in2_match = self.in2_regex.match(line) |
|
385 | 379 | in2_match_rstrip = self.in2_regex_rstrip.match(line) |
|
386 | 380 | if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \ |
|
387 | 381 | in2_match_rstrip: |
|
388 | 382 | end_input = True |
|
389 | 383 | else: |
|
390 | 384 | end_input = False |
|
391 | 385 | if end_input and self.mode != 'tb': |
|
392 | 386 | # Only look for an end of input when not in tb mode. |
|
393 | 387 | # An ellipsis could appear within the traceback. |
|
394 | 388 | mode = 'output' |
|
395 | 389 | code = u'' |
|
396 | 390 | insertion = (0, Generic.Prompt, line) |
|
397 | 391 | return mode, code, insertion |
|
398 | 392 | |
|
399 | 393 | # Check for output prompt |
|
400 | 394 | out_match = self.out_regex.match(line) |
|
401 | 395 | out_match_rstrip = self.out_regex_rstrip.match(line) |
|
402 | 396 | if out_match or out_match_rstrip: |
|
403 | 397 | mode = 'output' |
|
404 | 398 | if out_match: |
|
405 | 399 | idx = out_match.end() |
|
406 | 400 | else: |
|
407 | 401 | idx = out_match_rstrip.end() |
|
408 | 402 | code = line[idx:] |
|
409 | 403 | # Use the 'heading' token for output. We cannot use Generic.Error |
|
410 | 404 | # since it would conflict with exceptions. |
|
411 | 405 | insertion = (0, Generic.Heading, line[:idx]) |
|
412 | 406 | return mode, code, insertion |
|
413 | 407 | |
|
414 | 408 | |
|
415 | 409 | # Check for input or continuation prompt (non stripped version) |
|
416 | 410 | in1_match = self.in1_regex.match(line) |
|
417 | 411 | if in1_match or (in2_match and self.mode != 'tb'): |
|
418 | 412 | # New input or when not in tb, continued input. |
|
419 | 413 | # We do not check for continued input when in tb since it is |
|
420 | 414 | # allowable to replace a long stack with an ellipsis. |
|
421 | 415 | mode = 'input' |
|
422 | 416 | if in1_match: |
|
423 | 417 | idx = in1_match.end() |
|
424 | 418 | else: # in2_match |
|
425 | 419 | idx = in2_match.end() |
|
426 | 420 | code = line[idx:] |
|
427 | 421 | insertion = (0, Generic.Prompt, line[:idx]) |
|
428 | 422 | return mode, code, insertion |
|
429 | 423 | |
|
430 | 424 | # Check for input or continuation prompt (stripped version) |
|
431 | 425 | in1_match_rstrip = self.in1_regex_rstrip.match(line) |
|
432 | 426 | if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'): |
|
433 | 427 | # New input or when not in tb, continued input. |
|
434 | 428 | # We do not check for continued input when in tb since it is |
|
435 | 429 | # allowable to replace a long stack with an ellipsis. |
|
436 | 430 | mode = 'input' |
|
437 | 431 | if in1_match_rstrip: |
|
438 | 432 | idx = in1_match_rstrip.end() |
|
439 | 433 | else: # in2_match |
|
440 | 434 | idx = in2_match_rstrip.end() |
|
441 | 435 | code = line[idx:] |
|
442 | 436 | insertion = (0, Generic.Prompt, line[:idx]) |
|
443 | 437 | return mode, code, insertion |
|
444 | 438 | |
|
445 | 439 | # Check for traceback |
|
446 | 440 | if self.ipytb_start.match(line): |
|
447 | 441 | mode = 'tb' |
|
448 | 442 | code = line |
|
449 | 443 | insertion = None |
|
450 | 444 | return mode, code, insertion |
|
451 | 445 | |
|
452 | 446 | # All other stuff... |
|
453 | 447 | if self.mode in ('input', 'output'): |
|
454 | 448 | # We assume all other text is output. Multiline input that |
|
455 | 449 | # does not use the continuation marker cannot be detected. |
|
456 | 450 | # For example, the 3 in the following is clearly output: |
|
457 | 451 | # |
|
458 | 452 | # In [1]: print 3 |
|
459 | 453 | # 3 |
|
460 | 454 | # |
|
461 | 455 | # But the following second line is part of the input: |
|
462 | 456 | # |
|
463 | 457 | # In [2]: while True: |
|
464 | 458 | # print True |
|
465 | 459 | # |
|
466 | 460 | # In both cases, the 2nd line will be 'output'. |
|
467 | 461 | # |
|
468 | 462 | mode = 'output' |
|
469 | 463 | else: |
|
470 | 464 | mode = 'tb' |
|
471 | 465 | |
|
472 | 466 | code = line |
|
473 | 467 | insertion = None |
|
474 | 468 | |
|
475 | 469 | return mode, code, insertion |
|
476 | 470 | |
|
477 | 471 | def get_tokens_unprocessed(self, text): |
|
478 | 472 | self.reset() |
|
479 | 473 | for match in line_re.finditer(text): |
|
480 | 474 | line = match.group() |
|
481 | 475 | mode, code, insertion = self.get_mci(line) |
|
482 | 476 | |
|
483 | 477 | if mode != self.mode: |
|
484 | 478 | # Yield buffered tokens before transitioning to new mode. |
|
485 | 479 | for token in self.buffered_tokens(): |
|
486 | 480 | yield token |
|
487 | 481 | self.mode = mode |
|
488 | 482 | |
|
489 | 483 | if insertion: |
|
490 | 484 | self.insertions.append((len(self.buffer), [insertion])) |
|
491 | 485 | self.buffer += code |
|
492 | 486 | |
|
493 | 487 | for token in self.buffered_tokens(): |
|
494 | 488 | yield token |
|
495 | 489 | |
|
496 | 490 | class IPyLexer(Lexer): |
|
497 | 491 | r""" |
|
498 | 492 | Primary lexer for all IPython-like code. |
|
499 | 493 | |
|
500 | 494 | This is a simple helper lexer. If the first line of the text begins with |
|
501 | 495 | "In \[[0-9]+\]:", then the entire text is parsed with an IPython console |
|
502 | 496 | lexer. If not, then the entire text is parsed with an IPython lexer. |
|
503 | 497 | |
|
504 | 498 | The goal is to reduce the number of lexers that are registered |
|
505 | 499 | with Pygments. |
|
506 | 500 | |
|
507 | 501 | """ |
|
508 | 502 | name = 'IPy session' |
|
509 | 503 | aliases = ['ipy'] |
|
510 | 504 | |
|
511 | 505 | def __init__(self, **options): |
|
512 | 506 | self.python3 = get_bool_opt(options, 'python3', False) |
|
513 | 507 | if self.python3: |
|
514 | 508 | self.aliases = ['ipy3'] |
|
515 | 509 | else: |
|
516 | 510 | self.aliases = ['ipy2', 'ipy'] |
|
517 | 511 | |
|
518 | 512 | Lexer.__init__(self, **options) |
|
519 | 513 | |
|
520 | 514 | self.IPythonLexer = IPythonLexer(**options) |
|
521 | 515 | self.IPythonConsoleLexer = IPythonConsoleLexer(**options) |
|
522 | 516 | |
|
523 | 517 | def get_tokens_unprocessed(self, text): |
|
524 | 518 | # Search for the input prompt anywhere...this allows code blocks to |
|
525 | 519 | # begin with comments as well. |
|
526 | 520 | if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL): |
|
527 | 521 | lex = self.IPythonConsoleLexer |
|
528 | 522 | else: |
|
529 | 523 | lex = self.IPythonLexer |
|
530 | 524 | for token in lex.get_tokens_unprocessed(text): |
|
531 | 525 | yield token |
|
532 | 526 |
@@ -1,114 +1,114 b'' | |||
|
1 | 1 | """ |
|
2 | 2 | Password generation for the IPython notebook. |
|
3 | 3 | """ |
|
4 | 4 | #----------------------------------------------------------------------------- |
|
5 | 5 | # Imports |
|
6 | 6 | #----------------------------------------------------------------------------- |
|
7 | 7 | # Stdlib |
|
8 | 8 | import getpass |
|
9 | 9 | import hashlib |
|
10 | 10 | import random |
|
11 | 11 | |
|
12 | 12 | # Our own |
|
13 | 13 | from IPython.core.error import UsageError |
|
14 | 14 | from IPython.utils.py3compat import encode |
|
15 | 15 | |
|
16 | 16 | #----------------------------------------------------------------------------- |
|
17 | 17 | # Globals |
|
18 | 18 | #----------------------------------------------------------------------------- |
|
19 | 19 | |
|
20 | 20 | # Length of the salt in nr of hex chars, which implies salt_len * 4 |
|
21 | 21 | # bits of randomness. |
|
22 | 22 | salt_len = 12 |
|
23 | 23 | |
|
24 | 24 | #----------------------------------------------------------------------------- |
|
25 | 25 | # Functions |
|
26 | 26 | #----------------------------------------------------------------------------- |
|
27 | 27 | |
|
28 | 28 | def passwd(passphrase=None, algorithm='sha1'): |
|
29 | 29 | """Generate hashed password and salt for use in notebook configuration. |
|
30 | 30 | |
|
31 | 31 | In the notebook configuration, set `c.NotebookApp.password` to |
|
32 | 32 | the generated string. |
|
33 | 33 | |
|
34 | 34 | Parameters |
|
35 | 35 | ---------- |
|
36 | 36 | passphrase : str |
|
37 | 37 | Password to hash. If unspecified, the user is asked to input |
|
38 | 38 | and verify a password. |
|
39 | 39 | algorithm : str |
|
40 | 40 | Hashing algorithm to use (e.g, 'sha1' or any argument supported |
|
41 | 41 | by :func:`hashlib.new`). |
|
42 | 42 | |
|
43 | 43 | Returns |
|
44 | 44 | ------- |
|
45 | 45 | hashed_passphrase : str |
|
46 | 46 | Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'. |
|
47 | 47 | |
|
48 | 48 | Examples |
|
49 | 49 | -------- |
|
50 | 50 | >>> passwd('mypassword') |
|
51 | 'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12' | |
|
51 | 'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12' # random | |
|
52 | 52 | |
|
53 | 53 | """ |
|
54 | 54 | if passphrase is None: |
|
55 | 55 | for i in range(3): |
|
56 | 56 | p0 = getpass.getpass('Enter password: ') |
|
57 | 57 | p1 = getpass.getpass('Verify password: ') |
|
58 | 58 | if p0 == p1: |
|
59 | 59 | passphrase = p0 |
|
60 | 60 | break |
|
61 | 61 | else: |
|
62 | 62 | print('Passwords do not match.') |
|
63 | 63 | else: |
|
64 | 64 | raise UsageError('No matching passwords found. Giving up.') |
|
65 | 65 | |
|
66 | 66 | h = hashlib.new(algorithm) |
|
67 | 67 | salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len) |
|
68 | 68 | h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii')) |
|
69 | 69 | |
|
70 | 70 | return ':'.join((algorithm, salt, h.hexdigest())) |
|
71 | 71 | |
|
72 | 72 | |
|
73 | 73 | def passwd_check(hashed_passphrase, passphrase): |
|
74 | 74 | """Verify that a given passphrase matches its hashed version. |
|
75 | 75 | |
|
76 | 76 | Parameters |
|
77 | 77 | ---------- |
|
78 | 78 | hashed_passphrase : str |
|
79 | 79 | Hashed password, in the format returned by `passwd`. |
|
80 | 80 | passphrase : str |
|
81 | 81 | Passphrase to validate. |
|
82 | 82 | |
|
83 | 83 | Returns |
|
84 | 84 | ------- |
|
85 | 85 | valid : bool |
|
86 | 86 | True if the passphrase matches the hash. |
|
87 | 87 | |
|
88 | 88 | Examples |
|
89 | 89 | -------- |
|
90 | 90 | >>> from IPython.lib.security import passwd_check |
|
91 | 91 | >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a', |
|
92 | 92 | ... 'mypassword') |
|
93 | 93 | True |
|
94 | 94 | |
|
95 | 95 | >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a', |
|
96 | 96 | ... 'anotherpassword') |
|
97 | 97 | False |
|
98 | 98 | """ |
|
99 | 99 | try: |
|
100 | 100 | algorithm, salt, pw_digest = hashed_passphrase.split(':', 2) |
|
101 | 101 | except (ValueError, TypeError): |
|
102 | 102 | return False |
|
103 | 103 | |
|
104 | 104 | try: |
|
105 | 105 | h = hashlib.new(algorithm) |
|
106 | 106 | except ValueError: |
|
107 | 107 | return False |
|
108 | 108 | |
|
109 | 109 | if len(pw_digest) == 0: |
|
110 | 110 | return False |
|
111 | 111 | |
|
112 | 112 | h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii')) |
|
113 | 113 | |
|
114 | 114 | return h.hexdigest() == pw_digest |
@@ -1,157 +1,167 b'' | |||
|
1 | 1 | """Simple example using doctests. |
|
2 | 2 | |
|
3 | 3 | This file just contains doctests both using plain python and IPython prompts. |
|
4 | 4 | All tests should be loaded by nose. |
|
5 | 5 | """ |
|
6 | 6 | |
|
7 | import os | |
|
8 | ||
|
9 | ||
|
7 | 10 | def pyfunc(): |
|
8 | 11 | """Some pure python tests... |
|
9 | 12 | |
|
10 | 13 | >>> pyfunc() |
|
11 | 14 | 'pyfunc' |
|
12 | 15 | |
|
13 | 16 | >>> import os |
|
14 | 17 | |
|
15 | 18 | >>> 2+3 |
|
16 | 19 | 5 |
|
17 | 20 | |
|
18 | 21 | >>> for i in range(3): |
|
19 | 22 | ... print(i, end=' ') |
|
20 | 23 | ... print(i+1, end=' ') |
|
21 | 24 | ... |
|
22 | 25 | 0 1 1 2 2 3 |
|
23 | 26 | """ |
|
24 | 27 | return 'pyfunc' |
|
25 | 28 | |
|
26 | 29 | def ipfunc(): |
|
27 | 30 | """Some ipython tests... |
|
28 | 31 | |
|
29 | 32 | In [1]: import os |
|
30 | 33 | |
|
31 | 34 | In [3]: 2+3 |
|
32 | 35 | Out[3]: 5 |
|
33 | 36 | |
|
34 | 37 | In [26]: for i in range(3): |
|
35 | 38 | ....: print(i, end=' ') |
|
36 | 39 | ....: print(i+1, end=' ') |
|
37 | 40 | ....: |
|
38 | 41 | 0 1 1 2 2 3 |
|
39 | 42 | |
|
40 | 43 | |
|
41 | Examples that access the operating system work: | |
|
42 | ||
|
43 | In [1]: !echo hello | |
|
44 | hello | |
|
45 | ||
|
46 | In [2]: !echo hello > /tmp/foo_iptest | |
|
47 | ||
|
48 | In [3]: !cat /tmp/foo_iptest | |
|
49 | hello | |
|
50 | ||
|
51 | In [4]: rm -f /tmp/foo_iptest | |
|
52 | ||
|
53 | 44 | It's OK to use '_' for the last result, but do NOT try to use IPython's |
|
54 | 45 | numbered history of _NN outputs, since those won't exist under the |
|
55 | 46 | doctest environment: |
|
56 | 47 | |
|
57 | 48 | In [7]: 'hi' |
|
58 | 49 | Out[7]: 'hi' |
|
59 | 50 | |
|
60 | 51 | In [8]: print(repr(_)) |
|
61 | 52 | 'hi' |
|
62 | 53 | |
|
63 | 54 | In [7]: 3+4 |
|
64 | 55 | Out[7]: 7 |
|
65 | 56 | |
|
66 | 57 | In [8]: _+3 |
|
67 | 58 | Out[8]: 10 |
|
68 | 59 | |
|
69 | 60 | In [9]: ipfunc() |
|
70 | 61 | Out[9]: 'ipfunc' |
|
71 | 62 | """ |
|
72 | 63 | return 'ipfunc' |
|
73 | 64 | |
|
74 | 65 | |
|
66 | def ipos(): | |
|
67 | """Examples that access the operating system work: | |
|
68 | ||
|
69 | In [1]: !echo hello | |
|
70 | hello | |
|
71 | ||
|
72 | In [2]: !echo hello > /tmp/foo_iptest | |
|
73 | ||
|
74 | In [3]: !cat /tmp/foo_iptest | |
|
75 | hello | |
|
76 | ||
|
77 | In [4]: rm -f /tmp/foo_iptest | |
|
78 | """ | |
|
79 | pass | |
|
80 | ||
|
81 | ||
|
82 | ipos.__skip_doctest__ = os.name == "nt" | |
|
83 | ||
|
84 | ||
|
75 | 85 | def ranfunc(): |
|
76 | 86 | """A function with some random output. |
|
77 | 87 | |
|
78 | 88 | Normal examples are verified as usual: |
|
79 | 89 | >>> 1+3 |
|
80 | 90 | 4 |
|
81 | 91 | |
|
82 | 92 | But if you put '# random' in the output, it is ignored: |
|
83 | 93 | >>> 1+3 |
|
84 | 94 | junk goes here... # random |
|
85 | 95 | |
|
86 | 96 | >>> 1+2 |
|
87 | 97 | again, anything goes #random |
|
88 | 98 | if multiline, the random mark is only needed once. |
|
89 | 99 | |
|
90 | 100 | >>> 1+2 |
|
91 | 101 | You can also put the random marker at the end: |
|
92 | 102 | # random |
|
93 | 103 | |
|
94 | 104 | >>> 1+2 |
|
95 | 105 | # random |
|
96 | 106 | .. or at the beginning. |
|
97 | 107 | |
|
98 | 108 | More correct input is properly verified: |
|
99 | 109 | >>> ranfunc() |
|
100 | 110 | 'ranfunc' |
|
101 | 111 | """ |
|
102 | 112 | return 'ranfunc' |
|
103 | 113 | |
|
104 | 114 | |
|
105 | 115 | def random_all(): |
|
106 | 116 | """A function where we ignore the output of ALL examples. |
|
107 | 117 | |
|
108 | 118 | Examples: |
|
109 | 119 | |
|
110 | 120 | # all-random |
|
111 | 121 | |
|
112 | 122 | This mark tells the testing machinery that all subsequent examples should |
|
113 | 123 | be treated as random (ignoring their output). They are still executed, |
|
114 | 124 | so if a they raise an error, it will be detected as such, but their |
|
115 | 125 | output is completely ignored. |
|
116 | 126 | |
|
117 | 127 | >>> 1+3 |
|
118 | 128 | junk goes here... |
|
119 | 129 | |
|
120 | 130 | >>> 1+3 |
|
121 | 131 | klasdfj; |
|
122 | 132 | |
|
123 | 133 | >>> 1+2 |
|
124 | 134 | again, anything goes |
|
125 | 135 | blah... |
|
126 | 136 | """ |
|
127 | 137 | pass |
|
128 | 138 | |
|
129 | 139 | def iprand(): |
|
130 | 140 | """Some ipython tests with random output. |
|
131 | 141 | |
|
132 | 142 | In [7]: 3+4 |
|
133 | 143 | Out[7]: 7 |
|
134 | 144 | |
|
135 | 145 | In [8]: print('hello') |
|
136 | 146 | world # random |
|
137 | 147 | |
|
138 | 148 | In [9]: iprand() |
|
139 | 149 | Out[9]: 'iprand' |
|
140 | 150 | """ |
|
141 | 151 | return 'iprand' |
|
142 | 152 | |
|
143 | 153 | def iprand_all(): |
|
144 | 154 | """Some ipython tests with fully random output. |
|
145 | 155 | |
|
146 | 156 | # all-random |
|
147 | 157 | |
|
148 | 158 | In [7]: 1 |
|
149 | 159 | Out[7]: 99 |
|
150 | 160 | |
|
151 | 161 | In [8]: print('hello') |
|
152 | 162 | world |
|
153 | 163 | |
|
154 | 164 | In [9]: iprand_all() |
|
155 | 165 | Out[9]: 'junk' |
|
156 | 166 | """ |
|
157 | 167 | return 'iprand_all' |
@@ -1,30 +1,30 b'' | |||
|
1 | 1 | ================================= |
|
2 | 2 | Tests in example form - IPython |
|
3 | 3 | ================================= |
|
4 | 4 | |
|
5 | 5 | You can write text files with examples that use IPython prompts (as long as you |
|
6 | 6 | use the nose ipython doctest plugin), but you can not mix and match prompt |
|
7 | 7 | styles in a single file. That is, you either use all ``>>>`` prompts or all |
|
8 | 8 | IPython-style prompts. Your test suite *can* have both types, you just need to |
|
9 | 9 | put each type of example in a separate. Using IPython prompts, you can paste |
|
10 | 10 | directly from your session:: |
|
11 | 11 | |
|
12 | 12 | In [5]: s="Hello World" |
|
13 | 13 | |
|
14 | 14 | In [6]: s.upper() |
|
15 | 15 | Out[6]: 'HELLO WORLD' |
|
16 | 16 | |
|
17 | 17 | Another example:: |
|
18 | 18 | |
|
19 | 19 | In [8]: 1+3 |
|
20 | 20 | Out[8]: 4 |
|
21 | 21 | |
|
22 | 22 | Just like in IPython docstrings, you can use all IPython syntax and features:: |
|
23 | 23 | |
|
24 |
In [9]: !echo |
|
|
24 | In [9]: !echo hello | |
|
25 | 25 | hello |
|
26 | 26 | |
|
27 | 27 | In [10]: a='hi' |
|
28 | 28 | |
|
29 | 29 | In [11]: !echo $a |
|
30 | 30 | hi |
@@ -1,164 +1,168 b'' | |||
|
1 | 1 | """Tests for the decorators we've created for IPython. |
|
2 | 2 | """ |
|
3 | 3 | |
|
4 | 4 | # Module imports |
|
5 | 5 | # Std lib |
|
6 | 6 | import inspect |
|
7 | 7 | import sys |
|
8 | 8 | |
|
9 | 9 | # Third party |
|
10 | 10 | import nose.tools as nt |
|
11 | 11 | |
|
12 | 12 | # Our own |
|
13 | 13 | from IPython.testing import decorators as dec |
|
14 | from IPython.testing.skipdoctest import skip_doctest | |
|
14 | 15 | |
|
15 | 16 | #----------------------------------------------------------------------------- |
|
16 | 17 | # Utilities |
|
17 | 18 | |
|
18 | 19 | # Note: copied from OInspect, kept here so the testing stuff doesn't create |
|
19 | 20 | # circular dependencies and is easier to reuse. |
|
20 | 21 | def getargspec(obj): |
|
21 | 22 | """Get the names and default values of a function's arguments. |
|
22 | 23 | |
|
23 | 24 | A tuple of four things is returned: (args, varargs, varkw, defaults). |
|
24 | 25 | 'args' is a list of the argument names (it may contain nested lists). |
|
25 | 26 | 'varargs' and 'varkw' are the names of the * and ** arguments or None. |
|
26 | 27 | 'defaults' is an n-tuple of the default values of the last n arguments. |
|
27 | 28 | |
|
28 | 29 | Modified version of inspect.getargspec from the Python Standard |
|
29 | 30 | Library.""" |
|
30 | 31 | |
|
31 | 32 | if inspect.isfunction(obj): |
|
32 | 33 | func_obj = obj |
|
33 | 34 | elif inspect.ismethod(obj): |
|
34 | 35 | func_obj = obj.__func__ |
|
35 | 36 | else: |
|
36 | 37 | raise TypeError('arg is not a Python function') |
|
37 | 38 | args, varargs, varkw = inspect.getargs(func_obj.__code__) |
|
38 | 39 | return args, varargs, varkw, func_obj.__defaults__ |
|
39 | 40 | |
|
40 | 41 | #----------------------------------------------------------------------------- |
|
41 | 42 | # Testing functions |
|
42 | 43 | |
|
43 | 44 | @dec.as_unittest |
|
44 | 45 | def trivial(): |
|
45 | 46 | """A trivial test""" |
|
46 | 47 | pass |
|
47 | 48 | |
|
48 | 49 | |
|
49 | 50 | @dec.skip() |
|
50 | 51 | def test_deliberately_broken(): |
|
51 | 52 | """A deliberately broken test - we want to skip this one.""" |
|
52 | 53 | 1/0 |
|
53 | 54 | |
|
54 | 55 | @dec.skip('Testing the skip decorator') |
|
55 | 56 | def test_deliberately_broken2(): |
|
56 | 57 | """Another deliberately broken test - we want to skip this one.""" |
|
57 | 58 | 1/0 |
|
58 | 59 | |
|
59 | 60 | |
|
60 | 61 | # Verify that we can correctly skip the doctest for a function at will, but |
|
61 | 62 | # that the docstring itself is NOT destroyed by the decorator. |
|
63 | @skip_doctest | |
|
62 | 64 | def doctest_bad(x,y=1,**k): |
|
63 | 65 | """A function whose doctest we need to skip. |
|
64 | 66 | |
|
65 | 67 | >>> 1+1 |
|
66 | 68 | 3 |
|
67 | 69 | """ |
|
68 | 70 | print('x:',x) |
|
69 | 71 | print('y:',y) |
|
70 | 72 | print('k:',k) |
|
71 | 73 | |
|
72 | 74 | |
|
73 | 75 | def call_doctest_bad(): |
|
74 | 76 | """Check that we can still call the decorated functions. |
|
75 | 77 | |
|
76 | 78 | >>> doctest_bad(3,y=4) |
|
77 | 79 | x: 3 |
|
78 | 80 | y: 4 |
|
79 | 81 | k: {} |
|
80 | 82 | """ |
|
81 | 83 | pass |
|
82 | 84 | |
|
83 | 85 | |
|
84 | 86 | def test_skip_dt_decorator(): |
|
85 | 87 | """Doctest-skipping decorator should preserve the docstring. |
|
86 | 88 | """ |
|
87 | 89 | # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring! |
|
88 | 90 | check = """A function whose doctest we need to skip. |
|
89 | 91 | |
|
90 | 92 | >>> 1+1 |
|
91 | 93 | 3 |
|
92 | 94 | """ |
|
93 | 95 | # Fetch the docstring from doctest_bad after decoration. |
|
94 | 96 | val = doctest_bad.__doc__ |
|
95 | 97 | |
|
96 | 98 | nt.assert_equal(check,val,"doctest_bad docstrings don't match") |
|
97 | 99 | |
|
98 | 100 | |
|
99 | 101 | # Doctest skipping should work for class methods too |
|
100 | 102 | class FooClass(object): |
|
101 | 103 | """FooClass |
|
102 | 104 | |
|
103 | 105 | Example: |
|
104 | 106 | |
|
105 | 107 | >>> 1+1 |
|
106 | 108 | 2 |
|
107 | 109 | """ |
|
108 | 110 | |
|
111 | @skip_doctest | |
|
109 | 112 | def __init__(self,x): |
|
110 | 113 | """Make a FooClass. |
|
111 | 114 | |
|
112 | 115 | Example: |
|
113 | 116 | |
|
114 | 117 | >>> f = FooClass(3) |
|
115 | 118 | junk |
|
116 | 119 | """ |
|
117 | 120 | print('Making a FooClass.') |
|
118 | 121 | self.x = x |
|
119 | 122 | |
|
123 | @skip_doctest | |
|
120 | 124 | def bar(self,y): |
|
121 | 125 | """Example: |
|
122 | 126 | |
|
123 | 127 | >>> ff = FooClass(3) |
|
124 | 128 | >>> ff.bar(0) |
|
125 | 129 | boom! |
|
126 | 130 | >>> 1/0 |
|
127 | 131 | bam! |
|
128 | 132 | """ |
|
129 | 133 | return 1/y |
|
130 | 134 | |
|
131 | 135 | def baz(self,y): |
|
132 | 136 | """Example: |
|
133 | 137 | |
|
134 | 138 | >>> ff2 = FooClass(3) |
|
135 | 139 | Making a FooClass. |
|
136 | 140 | >>> ff2.baz(3) |
|
137 | 141 | True |
|
138 | 142 | """ |
|
139 | 143 | return self.x==y |
|
140 | 144 | |
|
141 | 145 | |
|
142 | 146 | def test_skip_dt_decorator2(): |
|
143 | 147 | """Doctest-skipping decorator should preserve function signature. |
|
144 | 148 | """ |
|
145 | 149 | # Hardcoded correct answer |
|
146 | 150 | dtargs = (['x', 'y'], None, 'k', (1,)) |
|
147 | 151 | # Introspect out the value |
|
148 | 152 | dtargsr = getargspec(doctest_bad) |
|
149 | 153 | assert dtargsr==dtargs, \ |
|
150 | 154 | "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,) |
|
151 | 155 | |
|
152 | 156 | |
|
153 | 157 | @dec.skip_linux |
|
154 | 158 | def test_linux(): |
|
155 | 159 | nt.assert_false(sys.platform.startswith('linux'),"This test can't run under linux") |
|
156 | 160 | |
|
157 | 161 | @dec.skip_win32 |
|
158 | 162 | def test_win32(): |
|
159 | 163 | nt.assert_not_equal(sys.platform,'win32',"This test can't run under windows") |
|
160 | 164 | |
|
161 | 165 | @dec.skip_osx |
|
162 | 166 | def test_osx(): |
|
163 | 167 | nt.assert_not_equal(sys.platform,'darwin',"This test can't run under osx") |
|
164 | 168 |
@@ -1,440 +1,440 b'' | |||
|
1 | 1 | # encoding: utf-8 |
|
2 | 2 | """ |
|
3 | 3 | Utilities for path handling. |
|
4 | 4 | """ |
|
5 | 5 | |
|
6 | 6 | # Copyright (c) IPython Development Team. |
|
7 | 7 | # Distributed under the terms of the Modified BSD License. |
|
8 | 8 | |
|
9 | 9 | import os |
|
10 | 10 | import sys |
|
11 | 11 | import errno |
|
12 | 12 | import shutil |
|
13 | 13 | import random |
|
14 | 14 | import glob |
|
15 | 15 | from warnings import warn |
|
16 | 16 | |
|
17 | 17 | from IPython.utils.process import system |
|
18 | 18 | from IPython.utils.decorators import undoc |
|
19 | 19 | |
|
20 | 20 | #----------------------------------------------------------------------------- |
|
21 | 21 | # Code |
|
22 | 22 | #----------------------------------------------------------------------------- |
|
23 | 23 | fs_encoding = sys.getfilesystemencoding() |
|
24 | 24 | |
|
25 | 25 | def _writable_dir(path): |
|
26 | 26 | """Whether `path` is a directory, to which the user has write access.""" |
|
27 | 27 | return os.path.isdir(path) and os.access(path, os.W_OK) |
|
28 | 28 | |
|
29 | 29 | if sys.platform == 'win32': |
|
30 | 30 | def _get_long_path_name(path): |
|
31 | 31 | """Get a long path name (expand ~) on Windows using ctypes. |
|
32 | 32 | |
|
33 | 33 | Examples |
|
34 | 34 | -------- |
|
35 | 35 | |
|
36 | >>> get_long_path_name('c:\\docume~1') | |
|
36 | >>> get_long_path_name('c:\\\\docume~1') | |
|
37 | 37 | 'c:\\\\Documents and Settings' |
|
38 | 38 | |
|
39 | 39 | """ |
|
40 | 40 | try: |
|
41 | 41 | import ctypes |
|
42 | 42 | except ImportError as e: |
|
43 | 43 | raise ImportError('you need to have ctypes installed for this to work') from e |
|
44 | 44 | _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW |
|
45 | 45 | _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, |
|
46 | 46 | ctypes.c_uint ] |
|
47 | 47 | |
|
48 | 48 | buf = ctypes.create_unicode_buffer(260) |
|
49 | 49 | rv = _GetLongPathName(path, buf, 260) |
|
50 | 50 | if rv == 0 or rv > 260: |
|
51 | 51 | return path |
|
52 | 52 | else: |
|
53 | 53 | return buf.value |
|
54 | 54 | else: |
|
55 | 55 | def _get_long_path_name(path): |
|
56 | 56 | """Dummy no-op.""" |
|
57 | 57 | return path |
|
58 | 58 | |
|
59 | 59 | |
|
60 | 60 | |
|
61 | 61 | def get_long_path_name(path): |
|
62 | 62 | """Expand a path into its long form. |
|
63 | 63 | |
|
64 | 64 | On Windows this expands any ~ in the paths. On other platforms, it is |
|
65 | 65 | a null operation. |
|
66 | 66 | """ |
|
67 | 67 | return _get_long_path_name(path) |
|
68 | 68 | |
|
69 | 69 | |
|
70 | 70 | def unquote_filename(name, win32=(sys.platform=='win32')): |
|
71 | 71 | """ On Windows, remove leading and trailing quotes from filenames. |
|
72 | 72 | |
|
73 | 73 | This function has been deprecated and should not be used any more: |
|
74 | 74 | unquoting is now taken care of by :func:`IPython.utils.process.arg_split`. |
|
75 | 75 | """ |
|
76 | 76 | warn("'unquote_filename' is deprecated since IPython 5.0 and should not " |
|
77 | 77 | "be used anymore", DeprecationWarning, stacklevel=2) |
|
78 | 78 | if win32: |
|
79 | 79 | if name.startswith(("'", '"')) and name.endswith(("'", '"')): |
|
80 | 80 | name = name[1:-1] |
|
81 | 81 | return name |
|
82 | 82 | |
|
83 | 83 | |
|
84 | 84 | def compress_user(path): |
|
85 | 85 | """Reverse of :func:`os.path.expanduser` |
|
86 | 86 | """ |
|
87 | 87 | home = os.path.expanduser('~') |
|
88 | 88 | if path.startswith(home): |
|
89 | 89 | path = "~" + path[len(home):] |
|
90 | 90 | return path |
|
91 | 91 | |
|
92 | 92 | def get_py_filename(name, force_win32=None): |
|
93 | 93 | """Return a valid python filename in the current directory. |
|
94 | 94 | |
|
95 | 95 | If the given name is not a file, it adds '.py' and searches again. |
|
96 | 96 | Raises IOError with an informative message if the file isn't found. |
|
97 | 97 | """ |
|
98 | 98 | |
|
99 | 99 | name = os.path.expanduser(name) |
|
100 | 100 | if force_win32 is not None: |
|
101 | 101 | warn("The 'force_win32' argument to 'get_py_filename' is deprecated " |
|
102 | 102 | "since IPython 5.0 and should not be used anymore", |
|
103 | 103 | DeprecationWarning, stacklevel=2) |
|
104 | 104 | if not os.path.isfile(name) and not name.endswith('.py'): |
|
105 | 105 | name += '.py' |
|
106 | 106 | if os.path.isfile(name): |
|
107 | 107 | return name |
|
108 | 108 | else: |
|
109 | 109 | raise IOError('File `%r` not found.' % name) |
|
110 | 110 | |
|
111 | 111 | |
|
112 | 112 | def filefind(filename: str, path_dirs=None) -> str: |
|
113 | 113 | """Find a file by looking through a sequence of paths. |
|
114 | 114 | |
|
115 | 115 | This iterates through a sequence of paths looking for a file and returns |
|
116 | 116 | the full, absolute path of the first occurrence of the file. If no set of |
|
117 | 117 | path dirs is given, the filename is tested as is, after running through |
|
118 | 118 | :func:`expandvars` and :func:`expanduser`. Thus a simple call:: |
|
119 | 119 | |
|
120 | 120 | filefind('myfile.txt') |
|
121 | 121 | |
|
122 | 122 | will find the file in the current working dir, but:: |
|
123 | 123 | |
|
124 | 124 | filefind('~/myfile.txt') |
|
125 | 125 | |
|
126 | 126 | Will find the file in the users home directory. This function does not |
|
127 | 127 | automatically try any paths, such as the cwd or the user's home directory. |
|
128 | 128 | |
|
129 | 129 | Parameters |
|
130 | 130 | ---------- |
|
131 | 131 | filename : str |
|
132 | 132 | The filename to look for. |
|
133 | 133 | path_dirs : str, None or sequence of str |
|
134 | 134 | The sequence of paths to look for the file in. If None, the filename |
|
135 | 135 | need to be absolute or be in the cwd. If a string, the string is |
|
136 | 136 | put into a sequence and the searched. If a sequence, walk through |
|
137 | 137 | each element and join with ``filename``, calling :func:`expandvars` |
|
138 | 138 | and :func:`expanduser` before testing for existence. |
|
139 | 139 | |
|
140 | 140 | Returns |
|
141 | 141 | ------- |
|
142 | 142 | path : str |
|
143 | 143 | returns absolute path to file. |
|
144 | 144 | |
|
145 | 145 | Raises |
|
146 | 146 | ------ |
|
147 | 147 | IOError |
|
148 | 148 | """ |
|
149 | 149 | |
|
150 | 150 | # If paths are quoted, abspath gets confused, strip them... |
|
151 | 151 | filename = filename.strip('"').strip("'") |
|
152 | 152 | # If the input is an absolute path, just check it exists |
|
153 | 153 | if os.path.isabs(filename) and os.path.isfile(filename): |
|
154 | 154 | return filename |
|
155 | 155 | |
|
156 | 156 | if path_dirs is None: |
|
157 | 157 | path_dirs = ("",) |
|
158 | 158 | elif isinstance(path_dirs, str): |
|
159 | 159 | path_dirs = (path_dirs,) |
|
160 | 160 | |
|
161 | 161 | for path in path_dirs: |
|
162 | 162 | if path == '.': path = os.getcwd() |
|
163 | 163 | testname = expand_path(os.path.join(path, filename)) |
|
164 | 164 | if os.path.isfile(testname): |
|
165 | 165 | return os.path.abspath(testname) |
|
166 | 166 | |
|
167 | 167 | raise IOError("File %r does not exist in any of the search paths: %r" % |
|
168 | 168 | (filename, path_dirs) ) |
|
169 | 169 | |
|
170 | 170 | |
|
171 | 171 | class HomeDirError(Exception): |
|
172 | 172 | pass |
|
173 | 173 | |
|
174 | 174 | |
|
175 | 175 | def get_home_dir(require_writable=False) -> str: |
|
176 | 176 | """Return the 'home' directory, as a unicode string. |
|
177 | 177 | |
|
178 | 178 | Uses os.path.expanduser('~'), and checks for writability. |
|
179 | 179 | |
|
180 | 180 | See stdlib docs for how this is determined. |
|
181 | 181 | For Python <3.8, $HOME is first priority on *ALL* platforms. |
|
182 | 182 | For Python >=3.8 on Windows, %HOME% is no longer considered. |
|
183 | 183 | |
|
184 | 184 | Parameters |
|
185 | 185 | ---------- |
|
186 | 186 | require_writable : bool [default: False] |
|
187 | 187 | if True: |
|
188 | 188 | guarantees the return value is a writable directory, otherwise |
|
189 | 189 | raises HomeDirError |
|
190 | 190 | if False: |
|
191 | 191 | The path is resolved, but it is not guaranteed to exist or be writable. |
|
192 | 192 | """ |
|
193 | 193 | |
|
194 | 194 | homedir = os.path.expanduser('~') |
|
195 | 195 | # Next line will make things work even when /home/ is a symlink to |
|
196 | 196 | # /usr/home as it is on FreeBSD, for example |
|
197 | 197 | homedir = os.path.realpath(homedir) |
|
198 | 198 | |
|
199 | 199 | if not _writable_dir(homedir) and os.name == 'nt': |
|
200 | 200 | # expanduser failed, use the registry to get the 'My Documents' folder. |
|
201 | 201 | try: |
|
202 | 202 | import winreg as wreg |
|
203 | 203 | with wreg.OpenKey( |
|
204 | 204 | wreg.HKEY_CURRENT_USER, |
|
205 | 205 | r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" |
|
206 | 206 | ) as key: |
|
207 | 207 | homedir = wreg.QueryValueEx(key,'Personal')[0] |
|
208 | 208 | except: |
|
209 | 209 | pass |
|
210 | 210 | |
|
211 | 211 | if (not require_writable) or _writable_dir(homedir): |
|
212 | 212 | assert isinstance(homedir, str), "Homedir should be unicode not bytes" |
|
213 | 213 | return homedir |
|
214 | 214 | else: |
|
215 | 215 | raise HomeDirError('%s is not a writable dir, ' |
|
216 | 216 | 'set $HOME environment variable to override' % homedir) |
|
217 | 217 | |
|
218 | 218 | def get_xdg_dir(): |
|
219 | 219 | """Return the XDG_CONFIG_HOME, if it is defined and exists, else None. |
|
220 | 220 | |
|
221 | 221 | This is only for non-OS X posix (Linux,Unix,etc.) systems. |
|
222 | 222 | """ |
|
223 | 223 | |
|
224 | 224 | env = os.environ |
|
225 | 225 | |
|
226 | 226 | if os.name == 'posix' and sys.platform != 'darwin': |
|
227 | 227 | # Linux, Unix, AIX, etc. |
|
228 | 228 | # use ~/.config if empty OR not set |
|
229 | 229 | xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config') |
|
230 | 230 | if xdg and _writable_dir(xdg): |
|
231 | 231 | assert isinstance(xdg, str) |
|
232 | 232 | return xdg |
|
233 | 233 | |
|
234 | 234 | return None |
|
235 | 235 | |
|
236 | 236 | |
|
237 | 237 | def get_xdg_cache_dir(): |
|
238 | 238 | """Return the XDG_CACHE_HOME, if it is defined and exists, else None. |
|
239 | 239 | |
|
240 | 240 | This is only for non-OS X posix (Linux,Unix,etc.) systems. |
|
241 | 241 | """ |
|
242 | 242 | |
|
243 | 243 | env = os.environ |
|
244 | 244 | |
|
245 | 245 | if os.name == 'posix' and sys.platform != 'darwin': |
|
246 | 246 | # Linux, Unix, AIX, etc. |
|
247 | 247 | # use ~/.cache if empty OR not set |
|
248 | 248 | xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache') |
|
249 | 249 | if xdg and _writable_dir(xdg): |
|
250 | 250 | assert isinstance(xdg, str) |
|
251 | 251 | return xdg |
|
252 | 252 | |
|
253 | 253 | return None |
|
254 | 254 | |
|
255 | 255 | |
|
256 | 256 | @undoc |
|
257 | 257 | def get_ipython_dir(): |
|
258 | 258 | warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) |
|
259 | 259 | from IPython.paths import get_ipython_dir |
|
260 | 260 | return get_ipython_dir() |
|
261 | 261 | |
|
262 | 262 | @undoc |
|
263 | 263 | def get_ipython_cache_dir(): |
|
264 | 264 | warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) |
|
265 | 265 | from IPython.paths import get_ipython_cache_dir |
|
266 | 266 | return get_ipython_cache_dir() |
|
267 | 267 | |
|
268 | 268 | @undoc |
|
269 | 269 | def get_ipython_package_dir(): |
|
270 | 270 | warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) |
|
271 | 271 | from IPython.paths import get_ipython_package_dir |
|
272 | 272 | return get_ipython_package_dir() |
|
273 | 273 | |
|
274 | 274 | @undoc |
|
275 | 275 | def get_ipython_module_path(module_str): |
|
276 | 276 | warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) |
|
277 | 277 | from IPython.paths import get_ipython_module_path |
|
278 | 278 | return get_ipython_module_path(module_str) |
|
279 | 279 | |
|
280 | 280 | @undoc |
|
281 | 281 | def locate_profile(profile='default'): |
|
282 | 282 | warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2) |
|
283 | 283 | from IPython.paths import locate_profile |
|
284 | 284 | return locate_profile(profile=profile) |
|
285 | 285 | |
|
286 | 286 | def expand_path(s): |
|
287 | 287 | """Expand $VARS and ~names in a string, like a shell |
|
288 | 288 | |
|
289 | 289 | :Examples: |
|
290 | 290 | |
|
291 | 291 | In [2]: os.environ['FOO']='test' |
|
292 | 292 | |
|
293 | 293 | In [3]: expand_path('variable FOO is $FOO') |
|
294 | 294 | Out[3]: 'variable FOO is test' |
|
295 | 295 | """ |
|
296 | 296 | # This is a pretty subtle hack. When expand user is given a UNC path |
|
297 | 297 | # on Windows (\\server\share$\%username%), os.path.expandvars, removes |
|
298 | 298 | # the $ to get (\\server\share\%username%). I think it considered $ |
|
299 | 299 | # alone an empty var. But, we need the $ to remains there (it indicates |
|
300 | 300 | # a hidden share). |
|
301 | 301 | if os.name=='nt': |
|
302 | 302 | s = s.replace('$\\', 'IPYTHON_TEMP') |
|
303 | 303 | s = os.path.expandvars(os.path.expanduser(s)) |
|
304 | 304 | if os.name=='nt': |
|
305 | 305 | s = s.replace('IPYTHON_TEMP', '$\\') |
|
306 | 306 | return s |
|
307 | 307 | |
|
308 | 308 | |
|
309 | 309 | def unescape_glob(string): |
|
310 | 310 | """Unescape glob pattern in `string`.""" |
|
311 | 311 | def unescape(s): |
|
312 | 312 | for pattern in '*[]!?': |
|
313 | 313 | s = s.replace(r'\{0}'.format(pattern), pattern) |
|
314 | 314 | return s |
|
315 | 315 | return '\\'.join(map(unescape, string.split('\\\\'))) |
|
316 | 316 | |
|
317 | 317 | |
|
318 | 318 | def shellglob(args): |
|
319 | 319 | """ |
|
320 | 320 | Do glob expansion for each element in `args` and return a flattened list. |
|
321 | 321 | |
|
322 | 322 | Unmatched glob pattern will remain as-is in the returned list. |
|
323 | 323 | |
|
324 | 324 | """ |
|
325 | 325 | expanded = [] |
|
326 | 326 | # Do not unescape backslash in Windows as it is interpreted as |
|
327 | 327 | # path separator: |
|
328 | 328 | unescape = unescape_glob if sys.platform != 'win32' else lambda x: x |
|
329 | 329 | for a in args: |
|
330 | 330 | expanded.extend(glob.glob(a) or [unescape(a)]) |
|
331 | 331 | return expanded |
|
332 | 332 | |
|
333 | 333 | |
|
334 | 334 | def target_outdated(target,deps): |
|
335 | 335 | """Determine whether a target is out of date. |
|
336 | 336 | |
|
337 | 337 | target_outdated(target,deps) -> 1/0 |
|
338 | 338 | |
|
339 | 339 | deps: list of filenames which MUST exist. |
|
340 | 340 | target: single filename which may or may not exist. |
|
341 | 341 | |
|
342 | 342 | If target doesn't exist or is older than any file listed in deps, return |
|
343 | 343 | true, otherwise return false. |
|
344 | 344 | """ |
|
345 | 345 | try: |
|
346 | 346 | target_time = os.path.getmtime(target) |
|
347 | 347 | except os.error: |
|
348 | 348 | return 1 |
|
349 | 349 | for dep in deps: |
|
350 | 350 | dep_time = os.path.getmtime(dep) |
|
351 | 351 | if dep_time > target_time: |
|
352 | 352 | #print "For target",target,"Dep failed:",dep # dbg |
|
353 | 353 | #print "times (dep,tar):",dep_time,target_time # dbg |
|
354 | 354 | return 1 |
|
355 | 355 | return 0 |
|
356 | 356 | |
|
357 | 357 | |
|
358 | 358 | def target_update(target,deps,cmd): |
|
359 | 359 | """Update a target with a given command given a list of dependencies. |
|
360 | 360 | |
|
361 | 361 | target_update(target,deps,cmd) -> runs cmd if target is outdated. |
|
362 | 362 | |
|
363 | 363 | This is just a wrapper around target_outdated() which calls the given |
|
364 | 364 | command if target is outdated.""" |
|
365 | 365 | |
|
366 | 366 | if target_outdated(target,deps): |
|
367 | 367 | system(cmd) |
|
368 | 368 | |
|
369 | 369 | |
|
370 | 370 | ENOLINK = 1998 |
|
371 | 371 | |
|
372 | 372 | def link(src, dst): |
|
373 | 373 | """Hard links ``src`` to ``dst``, returning 0 or errno. |
|
374 | 374 | |
|
375 | 375 | Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't |
|
376 | 376 | supported by the operating system. |
|
377 | 377 | """ |
|
378 | 378 | |
|
379 | 379 | if not hasattr(os, "link"): |
|
380 | 380 | return ENOLINK |
|
381 | 381 | link_errno = 0 |
|
382 | 382 | try: |
|
383 | 383 | os.link(src, dst) |
|
384 | 384 | except OSError as e: |
|
385 | 385 | link_errno = e.errno |
|
386 | 386 | return link_errno |
|
387 | 387 | |
|
388 | 388 | |
|
389 | 389 | def link_or_copy(src, dst): |
|
390 | 390 | """Attempts to hardlink ``src`` to ``dst``, copying if the link fails. |
|
391 | 391 | |
|
392 | 392 | Attempts to maintain the semantics of ``shutil.copy``. |
|
393 | 393 | |
|
394 | 394 | Because ``os.link`` does not overwrite files, a unique temporary file |
|
395 | 395 | will be used if the target already exists, then that file will be moved |
|
396 | 396 | into place. |
|
397 | 397 | """ |
|
398 | 398 | |
|
399 | 399 | if os.path.isdir(dst): |
|
400 | 400 | dst = os.path.join(dst, os.path.basename(src)) |
|
401 | 401 | |
|
402 | 402 | link_errno = link(src, dst) |
|
403 | 403 | if link_errno == errno.EEXIST: |
|
404 | 404 | if os.stat(src).st_ino == os.stat(dst).st_ino: |
|
405 | 405 | # dst is already a hard link to the correct file, so we don't need |
|
406 | 406 | # to do anything else. If we try to link and rename the file |
|
407 | 407 | # anyway, we get duplicate files - see http://bugs.python.org/issue21876 |
|
408 | 408 | return |
|
409 | 409 | |
|
410 | 410 | new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), ) |
|
411 | 411 | try: |
|
412 | 412 | link_or_copy(src, new_dst) |
|
413 | 413 | except: |
|
414 | 414 | try: |
|
415 | 415 | os.remove(new_dst) |
|
416 | 416 | except OSError: |
|
417 | 417 | pass |
|
418 | 418 | raise |
|
419 | 419 | os.rename(new_dst, dst) |
|
420 | 420 | elif link_errno != 0: |
|
421 | 421 | # Either link isn't supported, or the filesystem doesn't support |
|
422 | 422 | # linking, or 'src' and 'dst' are on different filesystems. |
|
423 | 423 | shutil.copy(src, dst) |
|
424 | 424 | |
|
425 | 425 | def ensure_dir_exists(path, mode=0o755): |
|
426 | 426 | """ensure that a directory exists |
|
427 | 427 | |
|
428 | 428 | If it doesn't exist, try to create it and protect against a race condition |
|
429 | 429 | if another process is doing the same. |
|
430 | 430 | |
|
431 | 431 | The default permissions are 755, which differ from os.makedirs default of 777. |
|
432 | 432 | """ |
|
433 | 433 | if not os.path.exists(path): |
|
434 | 434 | try: |
|
435 | 435 | os.makedirs(path, mode=mode) |
|
436 | 436 | except OSError as e: |
|
437 | 437 | if e.errno != errno.EEXIST: |
|
438 | 438 | raise |
|
439 | 439 | elif not os.path.isdir(path): |
|
440 | 440 | raise IOError("%r exists but is not a directory" % path) |
General Comments 0
You need to be logged in to leave comments.
Login now