##// END OF EJS Templates
Merge pull request #13213 from Kojoley/fix-bunch-of-doctests...
Matthias Bussonnier -
r26940:32497c8d merge
parent child Browse files
Show More
@@ -1,672 +1,674 b''
1 """Various display related classes.
1 """Various display related classes.
2
2
3 Authors : MinRK, gregcaporaso, dannystaple
3 Authors : MinRK, gregcaporaso, dannystaple
4 """
4 """
5 from html import escape as html_escape
5 from html import escape as html_escape
6 from os.path import exists, isfile, splitext, abspath, join, isdir
6 from os.path import exists, isfile, splitext, abspath, join, isdir
7 from os import walk, sep, fsdecode
7 from os import walk, sep, fsdecode
8
8
9 from IPython.core.display import DisplayObject, TextDisplayObject
9 from IPython.core.display import DisplayObject, TextDisplayObject
10
10
11 from typing import Tuple, Iterable
11 from typing import Tuple, Iterable
12
12
13 __all__ = ['Audio', 'IFrame', 'YouTubeVideo', 'VimeoVideo', 'ScribdDocument',
13 __all__ = ['Audio', 'IFrame', 'YouTubeVideo', 'VimeoVideo', 'ScribdDocument',
14 'FileLink', 'FileLinks', 'Code']
14 'FileLink', 'FileLinks', 'Code']
15
15
16
16
17 class Audio(DisplayObject):
17 class Audio(DisplayObject):
18 """Create an audio object.
18 """Create an audio object.
19
19
20 When this object is returned by an input cell or passed to the
20 When this object is returned by an input cell or passed to the
21 display function, it will result in Audio controls being displayed
21 display function, it will result in Audio controls being displayed
22 in the frontend (only works in the notebook).
22 in the frontend (only works in the notebook).
23
23
24 Parameters
24 Parameters
25 ----------
25 ----------
26 data : numpy array, list, unicode, str or bytes
26 data : numpy array, list, unicode, str or bytes
27 Can be one of
27 Can be one of
28
28
29 * Numpy 1d array containing the desired waveform (mono)
29 * Numpy 1d array containing the desired waveform (mono)
30 * Numpy 2d array containing waveforms for each channel.
30 * Numpy 2d array containing waveforms for each channel.
31 Shape=(NCHAN, NSAMPLES). For the standard channel order, see
31 Shape=(NCHAN, NSAMPLES). For the standard channel order, see
32 http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
32 http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
33 * List of float or integer representing the waveform (mono)
33 * List of float or integer representing the waveform (mono)
34 * String containing the filename
34 * String containing the filename
35 * Bytestring containing raw PCM data or
35 * Bytestring containing raw PCM data or
36 * URL pointing to a file on the web.
36 * URL pointing to a file on the web.
37
37
38 If the array option is used, the waveform will be normalized.
38 If the array option is used, the waveform will be normalized.
39
39
40 If a filename or url is used, the format support will be browser
40 If a filename or url is used, the format support will be browser
41 dependent.
41 dependent.
42 url : unicode
42 url : unicode
43 A URL to download the data from.
43 A URL to download the data from.
44 filename : unicode
44 filename : unicode
45 Path to a local file to load the data from.
45 Path to a local file to load the data from.
46 embed : boolean
46 embed : boolean
47 Should the audio data be embedded using a data URI (True) or should
47 Should the audio data be embedded using a data URI (True) or should
48 the original source be referenced. Set this to True if you want the
48 the original source be referenced. Set this to True if you want the
49 audio to playable later with no internet connection in the notebook.
49 audio to playable later with no internet connection in the notebook.
50
50
51 Default is `True`, unless the keyword argument `url` is set, then
51 Default is `True`, unless the keyword argument `url` is set, then
52 default value is `False`.
52 default value is `False`.
53 rate : integer
53 rate : integer
54 The sampling rate of the raw data.
54 The sampling rate of the raw data.
55 Only required when data parameter is being used as an array
55 Only required when data parameter is being used as an array
56 autoplay : bool
56 autoplay : bool
57 Set to True if the audio should immediately start playing.
57 Set to True if the audio should immediately start playing.
58 Default is `False`.
58 Default is `False`.
59 normalize : bool
59 normalize : bool
60 Whether audio should be normalized (rescaled) to the maximum possible
60 Whether audio should be normalized (rescaled) to the maximum possible
61 range. Default is `True`. When set to `False`, `data` must be between
61 range. Default is `True`. When set to `False`, `data` must be between
62 -1 and 1 (inclusive), otherwise an error is raised.
62 -1 and 1 (inclusive), otherwise an error is raised.
63 Applies only when `data` is a list or array of samples; other types of
63 Applies only when `data` is a list or array of samples; other types of
64 audio are never normalized.
64 audio are never normalized.
65
65
66 Examples
66 Examples
67 --------
67 --------
68
68
69 Generate a sound
69 Generate a sound
70
70
71 >>> import numpy as np
71 >>> import numpy as np
72 ... framerate = 44100
72 >>> framerate = 44100
73 ... t = np.linspace(0,5,framerate*5)
73 >>> t = np.linspace(0,5,framerate*5)
74 ... data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t)
74 >>> data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t)
75 ... Audio(data, rate=framerate)
75 >>> Audio(data, rate=framerate)
76 <IPython.lib.display.Audio object>
76
77
77 Can also do stereo or more channels
78 Can also do stereo or more channels
78
79
79 >>> dataleft = np.sin(2*np.pi*220*t)
80 >>> dataleft = np.sin(2*np.pi*220*t)
80 ... dataright = np.sin(2*np.pi*224*t)
81 >>> dataright = np.sin(2*np.pi*224*t)
81 ... Audio([dataleft, dataright], rate=framerate)
82 >>> Audio([dataleft, dataright], rate=framerate)
83 <IPython.lib.display.Audio object>
82
84
83 From URL:
85 From URL:
84
86
85 >>> Audio("http://www.nch.com.au/acm/8k16bitpcm.wav")
87 >>> Audio("http://www.nch.com.au/acm/8k16bitpcm.wav") # doctest: +SKIP
86 >>> Audio(url="http://www.w3schools.com/html/horse.ogg")
88 >>> Audio(url="http://www.w3schools.com/html/horse.ogg") # doctest: +SKIP
87
89
88 From a File:
90 From a File:
89
91
90 >>> Audio('/path/to/sound.wav')
92 >>> Audio('/path/to/sound.wav') # doctest: +SKIP
91 >>> Audio(filename='/path/to/sound.ogg')
93 >>> Audio(filename='/path/to/sound.ogg') # doctest: +SKIP
92
94
93 From Bytes:
95 From Bytes:
94
96
95 >>> Audio(b'RAW_WAV_DATA..')
97 >>> Audio(b'RAW_WAV_DATA..') # doctest: +SKIP
96 >>> Audio(data=b'RAW_WAV_DATA..')
98 >>> Audio(data=b'RAW_WAV_DATA..') # doctest: +SKIP
97
99
98 See Also
100 See Also
99 --------
101 --------
100 ipywidgets.Audio
102 ipywidgets.Audio
101
103
102 AUdio widget with more more flexibility and options.
104 AUdio widget with more more flexibility and options.
103
105
104 """
106 """
105 _read_flags = 'rb'
107 _read_flags = 'rb'
106
108
107 def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, autoplay=False, normalize=True, *,
109 def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, autoplay=False, normalize=True, *,
108 element_id=None):
110 element_id=None):
109 if filename is None and url is None and data is None:
111 if filename is None and url is None and data is None:
110 raise ValueError("No audio data found. Expecting filename, url, or data.")
112 raise ValueError("No audio data found. Expecting filename, url, or data.")
111 if embed is False and url is None:
113 if embed is False and url is None:
112 raise ValueError("No url found. Expecting url when embed=False")
114 raise ValueError("No url found. Expecting url when embed=False")
113
115
114 if url is not None and embed is not True:
116 if url is not None and embed is not True:
115 self.embed = False
117 self.embed = False
116 else:
118 else:
117 self.embed = True
119 self.embed = True
118 self.autoplay = autoplay
120 self.autoplay = autoplay
119 self.element_id = element_id
121 self.element_id = element_id
120 super(Audio, self).__init__(data=data, url=url, filename=filename)
122 super(Audio, self).__init__(data=data, url=url, filename=filename)
121
123
122 if self.data is not None and not isinstance(self.data, bytes):
124 if self.data is not None and not isinstance(self.data, bytes):
123 if rate is None:
125 if rate is None:
124 raise ValueError("rate must be specified when data is a numpy array or list of audio samples.")
126 raise ValueError("rate must be specified when data is a numpy array or list of audio samples.")
125 self.data = Audio._make_wav(data, rate, normalize)
127 self.data = Audio._make_wav(data, rate, normalize)
126
128
127 def reload(self):
129 def reload(self):
128 """Reload the raw data from file or URL."""
130 """Reload the raw data from file or URL."""
129 import mimetypes
131 import mimetypes
130 if self.embed:
132 if self.embed:
131 super(Audio, self).reload()
133 super(Audio, self).reload()
132
134
133 if self.filename is not None:
135 if self.filename is not None:
134 self.mimetype = mimetypes.guess_type(self.filename)[0]
136 self.mimetype = mimetypes.guess_type(self.filename)[0]
135 elif self.url is not None:
137 elif self.url is not None:
136 self.mimetype = mimetypes.guess_type(self.url)[0]
138 self.mimetype = mimetypes.guess_type(self.url)[0]
137 else:
139 else:
138 self.mimetype = "audio/wav"
140 self.mimetype = "audio/wav"
139
141
140 @staticmethod
142 @staticmethod
141 def _make_wav(data, rate, normalize):
143 def _make_wav(data, rate, normalize):
142 """ Transform a numpy array to a PCM bytestring """
144 """ Transform a numpy array to a PCM bytestring """
143 from io import BytesIO
145 from io import BytesIO
144 import wave
146 import wave
145
147
146 try:
148 try:
147 scaled, nchan = Audio._validate_and_normalize_with_numpy(data, normalize)
149 scaled, nchan = Audio._validate_and_normalize_with_numpy(data, normalize)
148 except ImportError:
150 except ImportError:
149 scaled, nchan = Audio._validate_and_normalize_without_numpy(data, normalize)
151 scaled, nchan = Audio._validate_and_normalize_without_numpy(data, normalize)
150
152
151 fp = BytesIO()
153 fp = BytesIO()
152 waveobj = wave.open(fp,mode='wb')
154 waveobj = wave.open(fp,mode='wb')
153 waveobj.setnchannels(nchan)
155 waveobj.setnchannels(nchan)
154 waveobj.setframerate(rate)
156 waveobj.setframerate(rate)
155 waveobj.setsampwidth(2)
157 waveobj.setsampwidth(2)
156 waveobj.setcomptype('NONE','NONE')
158 waveobj.setcomptype('NONE','NONE')
157 waveobj.writeframes(scaled)
159 waveobj.writeframes(scaled)
158 val = fp.getvalue()
160 val = fp.getvalue()
159 waveobj.close()
161 waveobj.close()
160
162
161 return val
163 return val
162
164
163 @staticmethod
165 @staticmethod
164 def _validate_and_normalize_with_numpy(data, normalize) -> Tuple[bytes, int]:
166 def _validate_and_normalize_with_numpy(data, normalize) -> Tuple[bytes, int]:
165 import numpy as np
167 import numpy as np
166
168
167 data = np.array(data, dtype=float)
169 data = np.array(data, dtype=float)
168 if len(data.shape) == 1:
170 if len(data.shape) == 1:
169 nchan = 1
171 nchan = 1
170 elif len(data.shape) == 2:
172 elif len(data.shape) == 2:
171 # In wave files,channels are interleaved. E.g.,
173 # In wave files,channels are interleaved. E.g.,
172 # "L1R1L2R2..." for stereo. See
174 # "L1R1L2R2..." for stereo. See
173 # http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
175 # http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
174 # for channel ordering
176 # for channel ordering
175 nchan = data.shape[0]
177 nchan = data.shape[0]
176 data = data.T.ravel()
178 data = data.T.ravel()
177 else:
179 else:
178 raise ValueError('Array audio input must be a 1D or 2D array')
180 raise ValueError('Array audio input must be a 1D or 2D array')
179
181
180 max_abs_value = np.max(np.abs(data))
182 max_abs_value = np.max(np.abs(data))
181 normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize)
183 normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize)
182 scaled = data / normalization_factor * 32767
184 scaled = data / normalization_factor * 32767
183 return scaled.astype("<h").tobytes(), nchan
185 return scaled.astype("<h").tobytes(), nchan
184
186
185 @staticmethod
187 @staticmethod
186 def _validate_and_normalize_without_numpy(data, normalize):
188 def _validate_and_normalize_without_numpy(data, normalize):
187 import array
189 import array
188 import sys
190 import sys
189
191
190 data = array.array('f', data)
192 data = array.array('f', data)
191
193
192 try:
194 try:
193 max_abs_value = float(max([abs(x) for x in data]))
195 max_abs_value = float(max([abs(x) for x in data]))
194 except TypeError as e:
196 except TypeError as e:
195 raise TypeError('Only lists of mono audio are '
197 raise TypeError('Only lists of mono audio are '
196 'supported if numpy is not installed') from e
198 'supported if numpy is not installed') from e
197
199
198 normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize)
200 normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize)
199 scaled = array.array('h', [int(x / normalization_factor * 32767) for x in data])
201 scaled = array.array('h', [int(x / normalization_factor * 32767) for x in data])
200 if sys.byteorder == 'big':
202 if sys.byteorder == 'big':
201 scaled.byteswap()
203 scaled.byteswap()
202 nchan = 1
204 nchan = 1
203 return scaled.tobytes(), nchan
205 return scaled.tobytes(), nchan
204
206
205 @staticmethod
207 @staticmethod
206 def _get_normalization_factor(max_abs_value, normalize):
208 def _get_normalization_factor(max_abs_value, normalize):
207 if not normalize and max_abs_value > 1:
209 if not normalize and max_abs_value > 1:
208 raise ValueError('Audio data must be between -1 and 1 when normalize=False.')
210 raise ValueError('Audio data must be between -1 and 1 when normalize=False.')
209 return max_abs_value if normalize else 1
211 return max_abs_value if normalize else 1
210
212
211 def _data_and_metadata(self):
213 def _data_and_metadata(self):
212 """shortcut for returning metadata with url information, if defined"""
214 """shortcut for returning metadata with url information, if defined"""
213 md = {}
215 md = {}
214 if self.url:
216 if self.url:
215 md['url'] = self.url
217 md['url'] = self.url
216 if md:
218 if md:
217 return self.data, md
219 return self.data, md
218 else:
220 else:
219 return self.data
221 return self.data
220
222
221 def _repr_html_(self):
223 def _repr_html_(self):
222 src = """
224 src = """
223 <audio {element_id} controls="controls" {autoplay}>
225 <audio {element_id} controls="controls" {autoplay}>
224 <source src="{src}" type="{type}" />
226 <source src="{src}" type="{type}" />
225 Your browser does not support the audio element.
227 Your browser does not support the audio element.
226 </audio>
228 </audio>
227 """
229 """
228 return src.format(src=self.src_attr(), type=self.mimetype, autoplay=self.autoplay_attr(),
230 return src.format(src=self.src_attr(), type=self.mimetype, autoplay=self.autoplay_attr(),
229 element_id=self.element_id_attr())
231 element_id=self.element_id_attr())
230
232
231 def src_attr(self):
233 def src_attr(self):
232 import base64
234 import base64
233 if self.embed and (self.data is not None):
235 if self.embed and (self.data is not None):
234 data = base64=base64.b64encode(self.data).decode('ascii')
236 data = base64=base64.b64encode(self.data).decode('ascii')
235 return """data:{type};base64,{base64}""".format(type=self.mimetype,
237 return """data:{type};base64,{base64}""".format(type=self.mimetype,
236 base64=data)
238 base64=data)
237 elif self.url is not None:
239 elif self.url is not None:
238 return self.url
240 return self.url
239 else:
241 else:
240 return ""
242 return ""
241
243
242 def autoplay_attr(self):
244 def autoplay_attr(self):
243 if(self.autoplay):
245 if(self.autoplay):
244 return 'autoplay="autoplay"'
246 return 'autoplay="autoplay"'
245 else:
247 else:
246 return ''
248 return ''
247
249
248 def element_id_attr(self):
250 def element_id_attr(self):
249 if (self.element_id):
251 if (self.element_id):
250 return 'id="{element_id}"'.format(element_id=self.element_id)
252 return 'id="{element_id}"'.format(element_id=self.element_id)
251 else:
253 else:
252 return ''
254 return ''
253
255
254 class IFrame(object):
256 class IFrame(object):
255 """
257 """
256 Generic class to embed an iframe in an IPython notebook
258 Generic class to embed an iframe in an IPython notebook
257 """
259 """
258
260
259 iframe = """
261 iframe = """
260 <iframe
262 <iframe
261 width="{width}"
263 width="{width}"
262 height="{height}"
264 height="{height}"
263 src="{src}{params}"
265 src="{src}{params}"
264 frameborder="0"
266 frameborder="0"
265 allowfullscreen
267 allowfullscreen
266 {extras}
268 {extras}
267 ></iframe>
269 ></iframe>
268 """
270 """
269
271
270 def __init__(self, src, width, height, extras: Iterable[str] = None, **kwargs):
272 def __init__(self, src, width, height, extras: Iterable[str] = None, **kwargs):
271 if extras is None:
273 if extras is None:
272 extras = []
274 extras = []
273
275
274 self.src = src
276 self.src = src
275 self.width = width
277 self.width = width
276 self.height = height
278 self.height = height
277 self.extras = extras
279 self.extras = extras
278 self.params = kwargs
280 self.params = kwargs
279
281
280 def _repr_html_(self):
282 def _repr_html_(self):
281 """return the embed iframe"""
283 """return the embed iframe"""
282 if self.params:
284 if self.params:
283 from urllib.parse import urlencode
285 from urllib.parse import urlencode
284 params = "?" + urlencode(self.params)
286 params = "?" + urlencode(self.params)
285 else:
287 else:
286 params = ""
288 params = ""
287 return self.iframe.format(
289 return self.iframe.format(
288 src=self.src,
290 src=self.src,
289 width=self.width,
291 width=self.width,
290 height=self.height,
292 height=self.height,
291 params=params,
293 params=params,
292 extras=" ".join(self.extras),
294 extras=" ".join(self.extras),
293 )
295 )
294
296
295
297
296 class YouTubeVideo(IFrame):
298 class YouTubeVideo(IFrame):
297 """Class for embedding a YouTube Video in an IPython session, based on its video id.
299 """Class for embedding a YouTube Video in an IPython session, based on its video id.
298
300
299 e.g. to embed the video from https://www.youtube.com/watch?v=foo , you would
301 e.g. to embed the video from https://www.youtube.com/watch?v=foo , you would
300 do::
302 do::
301
303
302 vid = YouTubeVideo("foo")
304 vid = YouTubeVideo("foo")
303 display(vid)
305 display(vid)
304
306
305 To start from 30 seconds::
307 To start from 30 seconds::
306
308
307 vid = YouTubeVideo("abc", start=30)
309 vid = YouTubeVideo("abc", start=30)
308 display(vid)
310 display(vid)
309
311
310 To calculate seconds from time as hours, minutes, seconds use
312 To calculate seconds from time as hours, minutes, seconds use
311 :class:`datetime.timedelta`::
313 :class:`datetime.timedelta`::
312
314
313 start=int(timedelta(hours=1, minutes=46, seconds=40).total_seconds())
315 start=int(timedelta(hours=1, minutes=46, seconds=40).total_seconds())
314
316
315 Other parameters can be provided as documented at
317 Other parameters can be provided as documented at
316 https://developers.google.com/youtube/player_parameters#Parameters
318 https://developers.google.com/youtube/player_parameters#Parameters
317
319
318 When converting the notebook using nbconvert, a jpeg representation of the video
320 When converting the notebook using nbconvert, a jpeg representation of the video
319 will be inserted in the document.
321 will be inserted in the document.
320 """
322 """
321
323
322 def __init__(self, id, width=400, height=300, allow_autoplay=False, **kwargs):
324 def __init__(self, id, width=400, height=300, allow_autoplay=False, **kwargs):
323 self.id=id
325 self.id=id
324 src = "https://www.youtube.com/embed/{0}".format(id)
326 src = "https://www.youtube.com/embed/{0}".format(id)
325 if allow_autoplay:
327 if allow_autoplay:
326 extras = list(kwargs.get("extras", [])) + ['allow="autoplay"']
328 extras = list(kwargs.get("extras", [])) + ['allow="autoplay"']
327 kwargs.update(autoplay=1, extras=extras)
329 kwargs.update(autoplay=1, extras=extras)
328 super(YouTubeVideo, self).__init__(src, width, height, **kwargs)
330 super(YouTubeVideo, self).__init__(src, width, height, **kwargs)
329
331
330 def _repr_jpeg_(self):
332 def _repr_jpeg_(self):
331 # Deferred import
333 # Deferred import
332 from urllib.request import urlopen
334 from urllib.request import urlopen
333
335
334 try:
336 try:
335 return urlopen("https://img.youtube.com/vi/{id}/hqdefault.jpg".format(id=self.id)).read()
337 return urlopen("https://img.youtube.com/vi/{id}/hqdefault.jpg".format(id=self.id)).read()
336 except IOError:
338 except IOError:
337 return None
339 return None
338
340
339 class VimeoVideo(IFrame):
341 class VimeoVideo(IFrame):
340 """
342 """
341 Class for embedding a Vimeo video in an IPython session, based on its video id.
343 Class for embedding a Vimeo video in an IPython session, based on its video id.
342 """
344 """
343
345
344 def __init__(self, id, width=400, height=300, **kwargs):
346 def __init__(self, id, width=400, height=300, **kwargs):
345 src="https://player.vimeo.com/video/{0}".format(id)
347 src="https://player.vimeo.com/video/{0}".format(id)
346 super(VimeoVideo, self).__init__(src, width, height, **kwargs)
348 super(VimeoVideo, self).__init__(src, width, height, **kwargs)
347
349
348 class ScribdDocument(IFrame):
350 class ScribdDocument(IFrame):
349 """
351 """
350 Class for embedding a Scribd document in an IPython session
352 Class for embedding a Scribd document in an IPython session
351
353
352 Use the start_page params to specify a starting point in the document
354 Use the start_page params to specify a starting point in the document
353 Use the view_mode params to specify display type one off scroll | slideshow | book
355 Use the view_mode params to specify display type one off scroll | slideshow | book
354
356
355 e.g to Display Wes' foundational paper about PANDAS in book mode from page 3
357 e.g to Display Wes' foundational paper about PANDAS in book mode from page 3
356
358
357 ScribdDocument(71048089, width=800, height=400, start_page=3, view_mode="book")
359 ScribdDocument(71048089, width=800, height=400, start_page=3, view_mode="book")
358 """
360 """
359
361
360 def __init__(self, id, width=400, height=300, **kwargs):
362 def __init__(self, id, width=400, height=300, **kwargs):
361 src="https://www.scribd.com/embeds/{0}/content".format(id)
363 src="https://www.scribd.com/embeds/{0}/content".format(id)
362 super(ScribdDocument, self).__init__(src, width, height, **kwargs)
364 super(ScribdDocument, self).__init__(src, width, height, **kwargs)
363
365
364 class FileLink(object):
366 class FileLink(object):
365 """Class for embedding a local file link in an IPython session, based on path
367 """Class for embedding a local file link in an IPython session, based on path
366
368
367 e.g. to embed a link that was generated in the IPython notebook as my/data.txt
369 e.g. to embed a link that was generated in the IPython notebook as my/data.txt
368
370
369 you would do::
371 you would do::
370
372
371 local_file = FileLink("my/data.txt")
373 local_file = FileLink("my/data.txt")
372 display(local_file)
374 display(local_file)
373
375
374 or in the HTML notebook, just::
376 or in the HTML notebook, just::
375
377
376 FileLink("my/data.txt")
378 FileLink("my/data.txt")
377 """
379 """
378
380
379 html_link_str = "<a href='%s' target='_blank'>%s</a>"
381 html_link_str = "<a href='%s' target='_blank'>%s</a>"
380
382
381 def __init__(self,
383 def __init__(self,
382 path,
384 path,
383 url_prefix='',
385 url_prefix='',
384 result_html_prefix='',
386 result_html_prefix='',
385 result_html_suffix='<br>'):
387 result_html_suffix='<br>'):
386 """
388 """
387 Parameters
389 Parameters
388 ----------
390 ----------
389 path : str
391 path : str
390 path to the file or directory that should be formatted
392 path to the file or directory that should be formatted
391 url_prefix : str
393 url_prefix : str
392 prefix to be prepended to all files to form a working link [default:
394 prefix to be prepended to all files to form a working link [default:
393 '']
395 '']
394 result_html_prefix : str
396 result_html_prefix : str
395 text to append to beginning to link [default: '']
397 text to append to beginning to link [default: '']
396 result_html_suffix : str
398 result_html_suffix : str
397 text to append at the end of link [default: '<br>']
399 text to append at the end of link [default: '<br>']
398 """
400 """
399 if isdir(path):
401 if isdir(path):
400 raise ValueError("Cannot display a directory using FileLink. "
402 raise ValueError("Cannot display a directory using FileLink. "
401 "Use FileLinks to display '%s'." % path)
403 "Use FileLinks to display '%s'." % path)
402 self.path = fsdecode(path)
404 self.path = fsdecode(path)
403 self.url_prefix = url_prefix
405 self.url_prefix = url_prefix
404 self.result_html_prefix = result_html_prefix
406 self.result_html_prefix = result_html_prefix
405 self.result_html_suffix = result_html_suffix
407 self.result_html_suffix = result_html_suffix
406
408
407 def _format_path(self):
409 def _format_path(self):
408 fp = ''.join([self.url_prefix, html_escape(self.path)])
410 fp = ''.join([self.url_prefix, html_escape(self.path)])
409 return ''.join([self.result_html_prefix,
411 return ''.join([self.result_html_prefix,
410 self.html_link_str % \
412 self.html_link_str % \
411 (fp, html_escape(self.path, quote=False)),
413 (fp, html_escape(self.path, quote=False)),
412 self.result_html_suffix])
414 self.result_html_suffix])
413
415
414 def _repr_html_(self):
416 def _repr_html_(self):
415 """return html link to file
417 """return html link to file
416 """
418 """
417 if not exists(self.path):
419 if not exists(self.path):
418 return ("Path (<tt>%s</tt>) doesn't exist. "
420 return ("Path (<tt>%s</tt>) doesn't exist. "
419 "It may still be in the process of "
421 "It may still be in the process of "
420 "being generated, or you may have the "
422 "being generated, or you may have the "
421 "incorrect path." % self.path)
423 "incorrect path." % self.path)
422
424
423 return self._format_path()
425 return self._format_path()
424
426
425 def __repr__(self):
427 def __repr__(self):
426 """return absolute path to file
428 """return absolute path to file
427 """
429 """
428 return abspath(self.path)
430 return abspath(self.path)
429
431
430 class FileLinks(FileLink):
432 class FileLinks(FileLink):
431 """Class for embedding local file links in an IPython session, based on path
433 """Class for embedding local file links in an IPython session, based on path
432
434
433 e.g. to embed links to files that were generated in the IPython notebook
435 e.g. to embed links to files that were generated in the IPython notebook
434 under ``my/data``, you would do::
436 under ``my/data``, you would do::
435
437
436 local_files = FileLinks("my/data")
438 local_files = FileLinks("my/data")
437 display(local_files)
439 display(local_files)
438
440
439 or in the HTML notebook, just::
441 or in the HTML notebook, just::
440
442
441 FileLinks("my/data")
443 FileLinks("my/data")
442 """
444 """
443 def __init__(self,
445 def __init__(self,
444 path,
446 path,
445 url_prefix='',
447 url_prefix='',
446 included_suffixes=None,
448 included_suffixes=None,
447 result_html_prefix='',
449 result_html_prefix='',
448 result_html_suffix='<br>',
450 result_html_suffix='<br>',
449 notebook_display_formatter=None,
451 notebook_display_formatter=None,
450 terminal_display_formatter=None,
452 terminal_display_formatter=None,
451 recursive=True):
453 recursive=True):
452 """
454 """
453 See :class:`FileLink` for the ``path``, ``url_prefix``,
455 See :class:`FileLink` for the ``path``, ``url_prefix``,
454 ``result_html_prefix`` and ``result_html_suffix`` parameters.
456 ``result_html_prefix`` and ``result_html_suffix`` parameters.
455
457
456 included_suffixes : list
458 included_suffixes : list
457 Filename suffixes to include when formatting output [default: include
459 Filename suffixes to include when formatting output [default: include
458 all files]
460 all files]
459
461
460 notebook_display_formatter : function
462 notebook_display_formatter : function
461 Used to format links for display in the notebook. See discussion of
463 Used to format links for display in the notebook. See discussion of
462 formatter functions below.
464 formatter functions below.
463
465
464 terminal_display_formatter : function
466 terminal_display_formatter : function
465 Used to format links for display in the terminal. See discussion of
467 Used to format links for display in the terminal. See discussion of
466 formatter functions below.
468 formatter functions below.
467
469
468 Formatter functions must be of the form::
470 Formatter functions must be of the form::
469
471
470 f(dirname, fnames, included_suffixes)
472 f(dirname, fnames, included_suffixes)
471
473
472 dirname : str
474 dirname : str
473 The name of a directory
475 The name of a directory
474 fnames : list
476 fnames : list
475 The files in that directory
477 The files in that directory
476 included_suffixes : list
478 included_suffixes : list
477 The file suffixes that should be included in the output (passing None
479 The file suffixes that should be included in the output (passing None
478 meansto include all suffixes in the output in the built-in formatters)
480 meansto include all suffixes in the output in the built-in formatters)
479 recursive : boolean
481 recursive : boolean
480 Whether to recurse into subdirectories. Default is True.
482 Whether to recurse into subdirectories. Default is True.
481
483
482 The function should return a list of lines that will be printed in the
484 The function should return a list of lines that will be printed in the
483 notebook (if passing notebook_display_formatter) or the terminal (if
485 notebook (if passing notebook_display_formatter) or the terminal (if
484 passing terminal_display_formatter). This function is iterated over for
486 passing terminal_display_formatter). This function is iterated over for
485 each directory in self.path. Default formatters are in place, can be
487 each directory in self.path. Default formatters are in place, can be
486 passed here to support alternative formatting.
488 passed here to support alternative formatting.
487
489
488 """
490 """
489 if isfile(path):
491 if isfile(path):
490 raise ValueError("Cannot display a file using FileLinks. "
492 raise ValueError("Cannot display a file using FileLinks. "
491 "Use FileLink to display '%s'." % path)
493 "Use FileLink to display '%s'." % path)
492 self.included_suffixes = included_suffixes
494 self.included_suffixes = included_suffixes
493 # remove trailing slashes for more consistent output formatting
495 # remove trailing slashes for more consistent output formatting
494 path = path.rstrip('/')
496 path = path.rstrip('/')
495
497
496 self.path = path
498 self.path = path
497 self.url_prefix = url_prefix
499 self.url_prefix = url_prefix
498 self.result_html_prefix = result_html_prefix
500 self.result_html_prefix = result_html_prefix
499 self.result_html_suffix = result_html_suffix
501 self.result_html_suffix = result_html_suffix
500
502
501 self.notebook_display_formatter = \
503 self.notebook_display_formatter = \
502 notebook_display_formatter or self._get_notebook_display_formatter()
504 notebook_display_formatter or self._get_notebook_display_formatter()
503 self.terminal_display_formatter = \
505 self.terminal_display_formatter = \
504 terminal_display_formatter or self._get_terminal_display_formatter()
506 terminal_display_formatter or self._get_terminal_display_formatter()
505
507
506 self.recursive = recursive
508 self.recursive = recursive
507
509
508 def _get_display_formatter(self,
510 def _get_display_formatter(self,
509 dirname_output_format,
511 dirname_output_format,
510 fname_output_format,
512 fname_output_format,
511 fp_format,
513 fp_format,
512 fp_cleaner=None):
514 fp_cleaner=None):
513 """ generate built-in formatter function
515 """ generate built-in formatter function
514
516
515 this is used to define both the notebook and terminal built-in
517 this is used to define both the notebook and terminal built-in
516 formatters as they only differ by some wrapper text for each entry
518 formatters as they only differ by some wrapper text for each entry
517
519
518 dirname_output_format: string to use for formatting directory
520 dirname_output_format: string to use for formatting directory
519 names, dirname will be substituted for a single "%s" which
521 names, dirname will be substituted for a single "%s" which
520 must appear in this string
522 must appear in this string
521 fname_output_format: string to use for formatting file names,
523 fname_output_format: string to use for formatting file names,
522 if a single "%s" appears in the string, fname will be substituted
524 if a single "%s" appears in the string, fname will be substituted
523 if two "%s" appear in the string, the path to fname will be
525 if two "%s" appear in the string, the path to fname will be
524 substituted for the first and fname will be substituted for the
526 substituted for the first and fname will be substituted for the
525 second
527 second
526 fp_format: string to use for formatting filepaths, must contain
528 fp_format: string to use for formatting filepaths, must contain
527 exactly two "%s" and the dirname will be substituted for the first
529 exactly two "%s" and the dirname will be substituted for the first
528 and fname will be substituted for the second
530 and fname will be substituted for the second
529 """
531 """
530 def f(dirname, fnames, included_suffixes=None):
532 def f(dirname, fnames, included_suffixes=None):
531 result = []
533 result = []
532 # begin by figuring out which filenames, if any,
534 # begin by figuring out which filenames, if any,
533 # are going to be displayed
535 # are going to be displayed
534 display_fnames = []
536 display_fnames = []
535 for fname in fnames:
537 for fname in fnames:
536 if (isfile(join(dirname,fname)) and
538 if (isfile(join(dirname,fname)) and
537 (included_suffixes is None or
539 (included_suffixes is None or
538 splitext(fname)[1] in included_suffixes)):
540 splitext(fname)[1] in included_suffixes)):
539 display_fnames.append(fname)
541 display_fnames.append(fname)
540
542
541 if len(display_fnames) == 0:
543 if len(display_fnames) == 0:
542 # if there are no filenames to display, don't print anything
544 # if there are no filenames to display, don't print anything
543 # (not even the directory name)
545 # (not even the directory name)
544 pass
546 pass
545 else:
547 else:
546 # otherwise print the formatted directory name followed by
548 # otherwise print the formatted directory name followed by
547 # the formatted filenames
549 # the formatted filenames
548 dirname_output_line = dirname_output_format % dirname
550 dirname_output_line = dirname_output_format % dirname
549 result.append(dirname_output_line)
551 result.append(dirname_output_line)
550 for fname in display_fnames:
552 for fname in display_fnames:
551 fp = fp_format % (dirname,fname)
553 fp = fp_format % (dirname,fname)
552 if fp_cleaner is not None:
554 if fp_cleaner is not None:
553 fp = fp_cleaner(fp)
555 fp = fp_cleaner(fp)
554 try:
556 try:
555 # output can include both a filepath and a filename...
557 # output can include both a filepath and a filename...
556 fname_output_line = fname_output_format % (fp, fname)
558 fname_output_line = fname_output_format % (fp, fname)
557 except TypeError:
559 except TypeError:
558 # ... or just a single filepath
560 # ... or just a single filepath
559 fname_output_line = fname_output_format % fname
561 fname_output_line = fname_output_format % fname
560 result.append(fname_output_line)
562 result.append(fname_output_line)
561 return result
563 return result
562 return f
564 return f
563
565
564 def _get_notebook_display_formatter(self,
566 def _get_notebook_display_formatter(self,
565 spacer="&nbsp;&nbsp;"):
567 spacer="&nbsp;&nbsp;"):
566 """ generate function to use for notebook formatting
568 """ generate function to use for notebook formatting
567 """
569 """
568 dirname_output_format = \
570 dirname_output_format = \
569 self.result_html_prefix + "%s/" + self.result_html_suffix
571 self.result_html_prefix + "%s/" + self.result_html_suffix
570 fname_output_format = \
572 fname_output_format = \
571 self.result_html_prefix + spacer + self.html_link_str + self.result_html_suffix
573 self.result_html_prefix + spacer + self.html_link_str + self.result_html_suffix
572 fp_format = self.url_prefix + '%s/%s'
574 fp_format = self.url_prefix + '%s/%s'
573 if sep == "\\":
575 if sep == "\\":
574 # Working on a platform where the path separator is "\", so
576 # Working on a platform where the path separator is "\", so
575 # must convert these to "/" for generating a URI
577 # must convert these to "/" for generating a URI
576 def fp_cleaner(fp):
578 def fp_cleaner(fp):
577 # Replace all occurrences of backslash ("\") with a forward
579 # Replace all occurrences of backslash ("\") with a forward
578 # slash ("/") - this is necessary on windows when a path is
580 # slash ("/") - this is necessary on windows when a path is
579 # provided as input, but we must link to a URI
581 # provided as input, but we must link to a URI
580 return fp.replace('\\','/')
582 return fp.replace('\\','/')
581 else:
583 else:
582 fp_cleaner = None
584 fp_cleaner = None
583
585
584 return self._get_display_formatter(dirname_output_format,
586 return self._get_display_formatter(dirname_output_format,
585 fname_output_format,
587 fname_output_format,
586 fp_format,
588 fp_format,
587 fp_cleaner)
589 fp_cleaner)
588
590
589 def _get_terminal_display_formatter(self,
591 def _get_terminal_display_formatter(self,
590 spacer=" "):
592 spacer=" "):
591 """ generate function to use for terminal formatting
593 """ generate function to use for terminal formatting
592 """
594 """
593 dirname_output_format = "%s/"
595 dirname_output_format = "%s/"
594 fname_output_format = spacer + "%s"
596 fname_output_format = spacer + "%s"
595 fp_format = '%s/%s'
597 fp_format = '%s/%s'
596
598
597 return self._get_display_formatter(dirname_output_format,
599 return self._get_display_formatter(dirname_output_format,
598 fname_output_format,
600 fname_output_format,
599 fp_format)
601 fp_format)
600
602
601 def _format_path(self):
603 def _format_path(self):
602 result_lines = []
604 result_lines = []
603 if self.recursive:
605 if self.recursive:
604 walked_dir = list(walk(self.path))
606 walked_dir = list(walk(self.path))
605 else:
607 else:
606 walked_dir = [next(walk(self.path))]
608 walked_dir = [next(walk(self.path))]
607 walked_dir.sort()
609 walked_dir.sort()
608 for dirname, subdirs, fnames in walked_dir:
610 for dirname, subdirs, fnames in walked_dir:
609 result_lines += self.notebook_display_formatter(dirname, fnames, self.included_suffixes)
611 result_lines += self.notebook_display_formatter(dirname, fnames, self.included_suffixes)
610 return '\n'.join(result_lines)
612 return '\n'.join(result_lines)
611
613
612 def __repr__(self):
614 def __repr__(self):
613 """return newline-separated absolute paths
615 """return newline-separated absolute paths
614 """
616 """
615 result_lines = []
617 result_lines = []
616 if self.recursive:
618 if self.recursive:
617 walked_dir = list(walk(self.path))
619 walked_dir = list(walk(self.path))
618 else:
620 else:
619 walked_dir = [next(walk(self.path))]
621 walked_dir = [next(walk(self.path))]
620 walked_dir.sort()
622 walked_dir.sort()
621 for dirname, subdirs, fnames in walked_dir:
623 for dirname, subdirs, fnames in walked_dir:
622 result_lines += self.terminal_display_formatter(dirname, fnames, self.included_suffixes)
624 result_lines += self.terminal_display_formatter(dirname, fnames, self.included_suffixes)
623 return '\n'.join(result_lines)
625 return '\n'.join(result_lines)
624
626
625
627
626 class Code(TextDisplayObject):
628 class Code(TextDisplayObject):
627 """Display syntax-highlighted source code.
629 """Display syntax-highlighted source code.
628
630
629 This uses Pygments to highlight the code for HTML and Latex output.
631 This uses Pygments to highlight the code for HTML and Latex output.
630
632
631 Parameters
633 Parameters
632 ----------
634 ----------
633 data : str
635 data : str
634 The code as a string
636 The code as a string
635 url : str
637 url : str
636 A URL to fetch the code from
638 A URL to fetch the code from
637 filename : str
639 filename : str
638 A local filename to load the code from
640 A local filename to load the code from
639 language : str
641 language : str
640 The short name of a Pygments lexer to use for highlighting.
642 The short name of a Pygments lexer to use for highlighting.
641 If not specified, it will guess the lexer based on the filename
643 If not specified, it will guess the lexer based on the filename
642 or the code. Available lexers: http://pygments.org/docs/lexers/
644 or the code. Available lexers: http://pygments.org/docs/lexers/
643 """
645 """
644 def __init__(self, data=None, url=None, filename=None, language=None):
646 def __init__(self, data=None, url=None, filename=None, language=None):
645 self.language = language
647 self.language = language
646 super().__init__(data=data, url=url, filename=filename)
648 super().__init__(data=data, url=url, filename=filename)
647
649
648 def _get_lexer(self):
650 def _get_lexer(self):
649 if self.language:
651 if self.language:
650 from pygments.lexers import get_lexer_by_name
652 from pygments.lexers import get_lexer_by_name
651 return get_lexer_by_name(self.language)
653 return get_lexer_by_name(self.language)
652 elif self.filename:
654 elif self.filename:
653 from pygments.lexers import get_lexer_for_filename
655 from pygments.lexers import get_lexer_for_filename
654 return get_lexer_for_filename(self.filename)
656 return get_lexer_for_filename(self.filename)
655 else:
657 else:
656 from pygments.lexers import guess_lexer
658 from pygments.lexers import guess_lexer
657 return guess_lexer(self.data)
659 return guess_lexer(self.data)
658
660
659 def __repr__(self):
661 def __repr__(self):
660 return self.data
662 return self.data
661
663
662 def _repr_html_(self):
664 def _repr_html_(self):
663 from pygments import highlight
665 from pygments import highlight
664 from pygments.formatters import HtmlFormatter
666 from pygments.formatters import HtmlFormatter
665 fmt = HtmlFormatter()
667 fmt = HtmlFormatter()
666 style = '<style>{}</style>'.format(fmt.get_style_defs('.output_html'))
668 style = '<style>{}</style>'.format(fmt.get_style_defs('.output_html'))
667 return style + highlight(self.data, self._get_lexer(), fmt)
669 return style + highlight(self.data, self._get_lexer(), fmt)
668
670
669 def _repr_latex_(self):
671 def _repr_latex_(self):
670 from pygments import highlight
672 from pygments import highlight
671 from pygments.formatters import LatexFormatter
673 from pygments.formatters import LatexFormatter
672 return highlight(self.data, self._get_lexer(), LatexFormatter())
674 return highlight(self.data, self._get_lexer(), LatexFormatter())
@@ -1,532 +1,526 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 Defines a variety of Pygments lexers for highlighting IPython code.
3 Defines a variety of Pygments lexers for highlighting IPython code.
4
4
5 This includes:
5 This includes:
6
6
7 IPythonLexer, IPython3Lexer
7 IPythonLexer, IPython3Lexer
8 Lexers for pure IPython (python + magic/shell commands)
8 Lexers for pure IPython (python + magic/shell commands)
9
9
10 IPythonPartialTracebackLexer, IPythonTracebackLexer
10 IPythonPartialTracebackLexer, IPythonTracebackLexer
11 Supports 2.x and 3.x via keyword `python3`. The partial traceback
11 Supports 2.x and 3.x via keyword `python3`. The partial traceback
12 lexer reads everything but the Python code appearing in a traceback.
12 lexer reads everything but the Python code appearing in a traceback.
13 The full lexer combines the partial lexer with an IPython lexer.
13 The full lexer combines the partial lexer with an IPython lexer.
14
14
15 IPythonConsoleLexer
15 IPythonConsoleLexer
16 A lexer for IPython console sessions, with support for tracebacks.
16 A lexer for IPython console sessions, with support for tracebacks.
17
17
18 IPyLexer
18 IPyLexer
19 A friendly lexer which examines the first line of text and from it,
19 A friendly lexer which examines the first line of text and from it,
20 decides whether to use an IPython lexer or an IPython console lexer.
20 decides whether to use an IPython lexer or an IPython console lexer.
21 This is probably the only lexer that needs to be explicitly added
21 This is probably the only lexer that needs to be explicitly added
22 to Pygments.
22 to Pygments.
23
23
24 """
24 """
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26 # Copyright (c) 2013, the IPython Development Team.
26 # Copyright (c) 2013, the IPython Development Team.
27 #
27 #
28 # Distributed under the terms of the Modified BSD License.
28 # Distributed under the terms of the Modified BSD License.
29 #
29 #
30 # The full license is in the file COPYING.txt, distributed with this software.
30 # The full license is in the file COPYING.txt, distributed with this software.
31 #-----------------------------------------------------------------------------
31 #-----------------------------------------------------------------------------
32
32
33 # Standard library
33 # Standard library
34 import re
34 import re
35
35
36 # Third party
36 # Third party
37 from pygments.lexers import (
37 from pygments.lexers import (
38 BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, PythonLexer,
38 BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, PythonLexer,
39 Python3Lexer, TexLexer)
39 Python3Lexer, TexLexer)
40 from pygments.lexer import (
40 from pygments.lexer import (
41 Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
41 Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
42 )
42 )
43 from pygments.token import (
43 from pygments.token import (
44 Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
44 Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
45 )
45 )
46 from pygments.util import get_bool_opt
46 from pygments.util import get_bool_opt
47
47
48 # Local
48 # Local
49
49
50 line_re = re.compile('.*?\n')
50 line_re = re.compile('.*?\n')
51
51
52 __all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer',
52 __all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer',
53 'IPythonPartialTracebackLexer', 'IPythonTracebackLexer',
53 'IPythonPartialTracebackLexer', 'IPythonTracebackLexer',
54 'IPythonConsoleLexer', 'IPyLexer']
54 'IPythonConsoleLexer', 'IPyLexer']
55
55
56
56
57 def build_ipy_lexer(python3):
57 def build_ipy_lexer(python3):
58 """Builds IPython lexers depending on the value of `python3`.
58 """Builds IPython lexers depending on the value of `python3`.
59
59
60 The lexer inherits from an appropriate Python lexer and then adds
60 The lexer inherits from an appropriate Python lexer and then adds
61 information about IPython specific keywords (i.e. magic commands,
61 information about IPython specific keywords (i.e. magic commands,
62 shell commands, etc.)
62 shell commands, etc.)
63
63
64 Parameters
64 Parameters
65 ----------
65 ----------
66 python3 : bool
66 python3 : bool
67 If `True`, then build an IPython lexer from a Python 3 lexer.
67 If `True`, then build an IPython lexer from a Python 3 lexer.
68
68
69 """
69 """
70 # It would be nice to have a single IPython lexer class which takes
70 # It would be nice to have a single IPython lexer class which takes
71 # a boolean `python3`. But since there are two Python lexer classes,
71 # a boolean `python3`. But since there are two Python lexer classes,
72 # we will also have two IPython lexer classes.
72 # we will also have two IPython lexer classes.
73 if python3:
73 if python3:
74 PyLexer = Python3Lexer
74 PyLexer = Python3Lexer
75 name = 'IPython3'
75 name = 'IPython3'
76 aliases = ['ipython3']
76 aliases = ['ipython3']
77 doc = """IPython3 Lexer"""
77 doc = """IPython3 Lexer"""
78 else:
78 else:
79 PyLexer = PythonLexer
79 PyLexer = PythonLexer
80 name = 'IPython'
80 name = 'IPython'
81 aliases = ['ipython2', 'ipython']
81 aliases = ['ipython2', 'ipython']
82 doc = """IPython Lexer"""
82 doc = """IPython Lexer"""
83
83
84 ipython_tokens = [
84 ipython_tokens = [
85 (r'(?s)(\s*)(%%capture)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
85 (r'(?s)(\s*)(%%capture)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
86 (r'(?s)(\s*)(%%debug)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
86 (r'(?s)(\s*)(%%debug)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
87 (r'(?is)(\s*)(%%html)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(HtmlLexer))),
87 (r'(?is)(\s*)(%%html)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(HtmlLexer))),
88 (r'(?s)(\s*)(%%javascript)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
88 (r'(?s)(\s*)(%%javascript)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
89 (r'(?s)(\s*)(%%js)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
89 (r'(?s)(\s*)(%%js)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
90 (r'(?s)(\s*)(%%latex)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(TexLexer))),
90 (r'(?s)(\s*)(%%latex)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(TexLexer))),
91 (r'(?s)(\s*)(%%perl)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PerlLexer))),
91 (r'(?s)(\s*)(%%perl)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PerlLexer))),
92 (r'(?s)(\s*)(%%prun)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
92 (r'(?s)(\s*)(%%prun)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
93 (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
93 (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
94 (r'(?s)(\s*)(%%python)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
94 (r'(?s)(\s*)(%%python)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
95 (r'(?s)(\s*)(%%python2)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PythonLexer))),
95 (r'(?s)(\s*)(%%python2)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PythonLexer))),
96 (r'(?s)(\s*)(%%python3)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))),
96 (r'(?s)(\s*)(%%python3)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))),
97 (r'(?s)(\s*)(%%ruby)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(RubyLexer))),
97 (r'(?s)(\s*)(%%ruby)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(RubyLexer))),
98 (r'(?s)(\s*)(%%time)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
98 (r'(?s)(\s*)(%%time)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
99 (r'(?s)(\s*)(%%timeit)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
99 (r'(?s)(\s*)(%%timeit)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
100 (r'(?s)(\s*)(%%writefile)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
100 (r'(?s)(\s*)(%%writefile)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
101 (r'(?s)(\s*)(%%file)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
101 (r'(?s)(\s*)(%%file)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
102 (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
102 (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
103 (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
103 (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
104 (r"(%%?)(\w+)(\?\??)$", bygroups(Operator, Keyword, Operator)),
104 (r"(%%?)(\w+)(\?\??)$", bygroups(Operator, Keyword, Operator)),
105 (r"\b(\?\??)(\s*)$", bygroups(Operator, Text)),
105 (r"\b(\?\??)(\s*)$", bygroups(Operator, Text)),
106 (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
106 (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
107 using(BashLexer), Text)),
107 using(BashLexer), Text)),
108 (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
108 (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
109 (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
109 (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
110 (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
110 (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
111 (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
111 (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
112 (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)),
112 (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)),
113 ]
113 ]
114
114
115 tokens = PyLexer.tokens.copy()
115 tokens = PyLexer.tokens.copy()
116 tokens['root'] = ipython_tokens + tokens['root']
116 tokens['root'] = ipython_tokens + tokens['root']
117
117
118 attrs = {'name': name, 'aliases': aliases, 'filenames': [],
118 attrs = {'name': name, 'aliases': aliases, 'filenames': [],
119 '__doc__': doc, 'tokens': tokens}
119 '__doc__': doc, 'tokens': tokens}
120
120
121 return type(name, (PyLexer,), attrs)
121 return type(name, (PyLexer,), attrs)
122
122
123
123
124 IPython3Lexer = build_ipy_lexer(python3=True)
124 IPython3Lexer = build_ipy_lexer(python3=True)
125 IPythonLexer = build_ipy_lexer(python3=False)
125 IPythonLexer = build_ipy_lexer(python3=False)
126
126
127
127
128 class IPythonPartialTracebackLexer(RegexLexer):
128 class IPythonPartialTracebackLexer(RegexLexer):
129 """
129 """
130 Partial lexer for IPython tracebacks.
130 Partial lexer for IPython tracebacks.
131
131
132 Handles all the non-python output.
132 Handles all the non-python output.
133
133
134 """
134 """
135 name = 'IPython Partial Traceback'
135 name = 'IPython Partial Traceback'
136
136
137 tokens = {
137 tokens = {
138 'root': [
138 'root': [
139 # Tracebacks for syntax errors have a different style.
139 # Tracebacks for syntax errors have a different style.
140 # For both types of tracebacks, we mark the first line with
140 # For both types of tracebacks, we mark the first line with
141 # Generic.Traceback. For syntax errors, we mark the filename
141 # Generic.Traceback. For syntax errors, we mark the filename
142 # as we mark the filenames for non-syntax tracebacks.
142 # as we mark the filenames for non-syntax tracebacks.
143 #
143 #
144 # These two regexps define how IPythonConsoleLexer finds a
144 # These two regexps define how IPythonConsoleLexer finds a
145 # traceback.
145 # traceback.
146 #
146 #
147 ## Non-syntax traceback
147 ## Non-syntax traceback
148 (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
148 (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
149 ## Syntax traceback
149 ## Syntax traceback
150 (r'^( File)(.*)(, line )(\d+\n)',
150 (r'^( File)(.*)(, line )(\d+\n)',
151 bygroups(Generic.Traceback, Name.Namespace,
151 bygroups(Generic.Traceback, Name.Namespace,
152 Generic.Traceback, Literal.Number.Integer)),
152 Generic.Traceback, Literal.Number.Integer)),
153
153
154 # (Exception Identifier)(Whitespace)(Traceback Message)
154 # (Exception Identifier)(Whitespace)(Traceback Message)
155 (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
155 (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
156 bygroups(Name.Exception, Generic.Whitespace, Text)),
156 bygroups(Name.Exception, Generic.Whitespace, Text)),
157 # (Module/Filename)(Text)(Callee)(Function Signature)
157 # (Module/Filename)(Text)(Callee)(Function Signature)
158 # Better options for callee and function signature?
158 # Better options for callee and function signature?
159 (r'(.*)( in )(.*)(\(.*\)\n)',
159 (r'(.*)( in )(.*)(\(.*\)\n)',
160 bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
160 bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
161 # Regular line: (Whitespace)(Line Number)(Python Code)
161 # Regular line: (Whitespace)(Line Number)(Python Code)
162 (r'(\s*?)(\d+)(.*?\n)',
162 (r'(\s*?)(\d+)(.*?\n)',
163 bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
163 bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
164 # Emphasized line: (Arrow)(Line Number)(Python Code)
164 # Emphasized line: (Arrow)(Line Number)(Python Code)
165 # Using Exception token so arrow color matches the Exception.
165 # Using Exception token so arrow color matches the Exception.
166 (r'(-*>?\s?)(\d+)(.*?\n)',
166 (r'(-*>?\s?)(\d+)(.*?\n)',
167 bygroups(Name.Exception, Literal.Number.Integer, Other)),
167 bygroups(Name.Exception, Literal.Number.Integer, Other)),
168 # (Exception Identifier)(Message)
168 # (Exception Identifier)(Message)
169 (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
169 (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
170 bygroups(Name.Exception, Text)),
170 bygroups(Name.Exception, Text)),
171 # Tag everything else as Other, will be handled later.
171 # Tag everything else as Other, will be handled later.
172 (r'.*\n', Other),
172 (r'.*\n', Other),
173 ],
173 ],
174 }
174 }
175
175
176
176
177 class IPythonTracebackLexer(DelegatingLexer):
177 class IPythonTracebackLexer(DelegatingLexer):
178 """
178 """
179 IPython traceback lexer.
179 IPython traceback lexer.
180
180
181 For doctests, the tracebacks can be snipped as much as desired with the
181 For doctests, the tracebacks can be snipped as much as desired with the
182 exception to the lines that designate a traceback. For non-syntax error
182 exception to the lines that designate a traceback. For non-syntax error
183 tracebacks, this is the line of hyphens. For syntax error tracebacks,
183 tracebacks, this is the line of hyphens. For syntax error tracebacks,
184 this is the line which lists the File and line number.
184 this is the line which lists the File and line number.
185
185
186 """
186 """
187 # The lexer inherits from DelegatingLexer. The "root" lexer is an
187 # The lexer inherits from DelegatingLexer. The "root" lexer is an
188 # appropriate IPython lexer, which depends on the value of the boolean
188 # appropriate IPython lexer, which depends on the value of the boolean
189 # `python3`. First, we parse with the partial IPython traceback lexer.
189 # `python3`. First, we parse with the partial IPython traceback lexer.
190 # Then, any code marked with the "Other" token is delegated to the root
190 # Then, any code marked with the "Other" token is delegated to the root
191 # lexer.
191 # lexer.
192 #
192 #
193 name = 'IPython Traceback'
193 name = 'IPython Traceback'
194 aliases = ['ipythontb']
194 aliases = ['ipythontb']
195
195
196 def __init__(self, **options):
196 def __init__(self, **options):
197 self.python3 = get_bool_opt(options, 'python3', False)
197 self.python3 = get_bool_opt(options, 'python3', False)
198 if self.python3:
198 if self.python3:
199 self.aliases = ['ipython3tb']
199 self.aliases = ['ipython3tb']
200 else:
200 else:
201 self.aliases = ['ipython2tb', 'ipythontb']
201 self.aliases = ['ipython2tb', 'ipythontb']
202
202
203 if self.python3:
203 if self.python3:
204 IPyLexer = IPython3Lexer
204 IPyLexer = IPython3Lexer
205 else:
205 else:
206 IPyLexer = IPythonLexer
206 IPyLexer = IPythonLexer
207
207
208 DelegatingLexer.__init__(self, IPyLexer,
208 DelegatingLexer.__init__(self, IPyLexer,
209 IPythonPartialTracebackLexer, **options)
209 IPythonPartialTracebackLexer, **options)
210
210
211 class IPythonConsoleLexer(Lexer):
211 class IPythonConsoleLexer(Lexer):
212 """
212 """
213 An IPython console lexer for IPython code-blocks and doctests, such as:
213 An IPython console lexer for IPython code-blocks and doctests, such as:
214
214
215 .. code-block:: rst
215 .. code-block:: rst
216
216
217 .. code-block:: ipythonconsole
217 .. code-block:: ipythonconsole
218
218
219 In [1]: a = 'foo'
219 In [1]: a = 'foo'
220
220
221 In [2]: a
221 In [2]: a
222 Out[2]: 'foo'
222 Out[2]: 'foo'
223
223
224 In [3]: print a
224 In [3]: print(a)
225 foo
225 foo
226
226
227 In [4]: 1 / 0
228
229
227
230 Support is also provided for IPython exceptions:
228 Support is also provided for IPython exceptions:
231
229
232 .. code-block:: rst
230 .. code-block:: rst
233
231
234 .. code-block:: ipythonconsole
232 .. code-block:: ipythonconsole
235
233
236 In [1]: raise Exception
234 In [1]: raise Exception
237
235 Traceback (most recent call last):
238 ---------------------------------------------------------------------------
236 ...
239 Exception Traceback (most recent call last)
237 Exception
240 <ipython-input-1-fca2ab0ca76b> in <module>
241 ----> 1 raise Exception
242
243 Exception:
244
238
245 """
239 """
246 name = 'IPython console session'
240 name = 'IPython console session'
247 aliases = ['ipythonconsole']
241 aliases = ['ipythonconsole']
248 mimetypes = ['text/x-ipython-console']
242 mimetypes = ['text/x-ipython-console']
249
243
250 # The regexps used to determine what is input and what is output.
244 # The regexps used to determine what is input and what is output.
251 # The default prompts for IPython are:
245 # The default prompts for IPython are:
252 #
246 #
253 # in = 'In [#]: '
247 # in = 'In [#]: '
254 # continuation = ' .D.: '
248 # continuation = ' .D.: '
255 # template = 'Out[#]: '
249 # template = 'Out[#]: '
256 #
250 #
257 # Where '#' is the 'prompt number' or 'execution count' and 'D'
251 # Where '#' is the 'prompt number' or 'execution count' and 'D'
258 # D is a number of dots matching the width of the execution count
252 # D is a number of dots matching the width of the execution count
259 #
253 #
260 in1_regex = r'In \[[0-9]+\]: '
254 in1_regex = r'In \[[0-9]+\]: '
261 in2_regex = r' \.\.+\.: '
255 in2_regex = r' \.\.+\.: '
262 out_regex = r'Out\[[0-9]+\]: '
256 out_regex = r'Out\[[0-9]+\]: '
263
257
264 #: The regex to determine when a traceback starts.
258 #: The regex to determine when a traceback starts.
265 ipytb_start = re.compile(r'^(\^C)?(-+\n)|^( File)(.*)(, line )(\d+\n)')
259 ipytb_start = re.compile(r'^(\^C)?(-+\n)|^( File)(.*)(, line )(\d+\n)')
266
260
267 def __init__(self, **options):
261 def __init__(self, **options):
268 """Initialize the IPython console lexer.
262 """Initialize the IPython console lexer.
269
263
270 Parameters
264 Parameters
271 ----------
265 ----------
272 python3 : bool
266 python3 : bool
273 If `True`, then the console inputs are parsed using a Python 3
267 If `True`, then the console inputs are parsed using a Python 3
274 lexer. Otherwise, they are parsed using a Python 2 lexer.
268 lexer. Otherwise, they are parsed using a Python 2 lexer.
275 in1_regex : RegexObject
269 in1_regex : RegexObject
276 The compiled regular expression used to detect the start
270 The compiled regular expression used to detect the start
277 of inputs. Although the IPython configuration setting may have a
271 of inputs. Although the IPython configuration setting may have a
278 trailing whitespace, do not include it in the regex. If `None`,
272 trailing whitespace, do not include it in the regex. If `None`,
279 then the default input prompt is assumed.
273 then the default input prompt is assumed.
280 in2_regex : RegexObject
274 in2_regex : RegexObject
281 The compiled regular expression used to detect the continuation
275 The compiled regular expression used to detect the continuation
282 of inputs. Although the IPython configuration setting may have a
276 of inputs. Although the IPython configuration setting may have a
283 trailing whitespace, do not include it in the regex. If `None`,
277 trailing whitespace, do not include it in the regex. If `None`,
284 then the default input prompt is assumed.
278 then the default input prompt is assumed.
285 out_regex : RegexObject
279 out_regex : RegexObject
286 The compiled regular expression used to detect outputs. If `None`,
280 The compiled regular expression used to detect outputs. If `None`,
287 then the default output prompt is assumed.
281 then the default output prompt is assumed.
288
282
289 """
283 """
290 self.python3 = get_bool_opt(options, 'python3', False)
284 self.python3 = get_bool_opt(options, 'python3', False)
291 if self.python3:
285 if self.python3:
292 self.aliases = ['ipython3console']
286 self.aliases = ['ipython3console']
293 else:
287 else:
294 self.aliases = ['ipython2console', 'ipythonconsole']
288 self.aliases = ['ipython2console', 'ipythonconsole']
295
289
296 in1_regex = options.get('in1_regex', self.in1_regex)
290 in1_regex = options.get('in1_regex', self.in1_regex)
297 in2_regex = options.get('in2_regex', self.in2_regex)
291 in2_regex = options.get('in2_regex', self.in2_regex)
298 out_regex = options.get('out_regex', self.out_regex)
292 out_regex = options.get('out_regex', self.out_regex)
299
293
300 # So that we can work with input and output prompts which have been
294 # So that we can work with input and output prompts which have been
301 # rstrip'd (possibly by editors) we also need rstrip'd variants. If
295 # rstrip'd (possibly by editors) we also need rstrip'd variants. If
302 # we do not do this, then such prompts will be tagged as 'output'.
296 # we do not do this, then such prompts will be tagged as 'output'.
303 # The reason can't just use the rstrip'd variants instead is because
297 # The reason can't just use the rstrip'd variants instead is because
304 # we want any whitespace associated with the prompt to be inserted
298 # we want any whitespace associated with the prompt to be inserted
305 # with the token. This allows formatted code to be modified so as hide
299 # with the token. This allows formatted code to be modified so as hide
306 # the appearance of prompts, with the whitespace included. One example
300 # the appearance of prompts, with the whitespace included. One example
307 # use of this is in copybutton.js from the standard lib Python docs.
301 # use of this is in copybutton.js from the standard lib Python docs.
308 in1_regex_rstrip = in1_regex.rstrip() + '\n'
302 in1_regex_rstrip = in1_regex.rstrip() + '\n'
309 in2_regex_rstrip = in2_regex.rstrip() + '\n'
303 in2_regex_rstrip = in2_regex.rstrip() + '\n'
310 out_regex_rstrip = out_regex.rstrip() + '\n'
304 out_regex_rstrip = out_regex.rstrip() + '\n'
311
305
312 # Compile and save them all.
306 # Compile and save them all.
313 attrs = ['in1_regex', 'in2_regex', 'out_regex',
307 attrs = ['in1_regex', 'in2_regex', 'out_regex',
314 'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
308 'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
315 for attr in attrs:
309 for attr in attrs:
316 self.__setattr__(attr, re.compile(locals()[attr]))
310 self.__setattr__(attr, re.compile(locals()[attr]))
317
311
318 Lexer.__init__(self, **options)
312 Lexer.__init__(self, **options)
319
313
320 if self.python3:
314 if self.python3:
321 pylexer = IPython3Lexer
315 pylexer = IPython3Lexer
322 tblexer = IPythonTracebackLexer
316 tblexer = IPythonTracebackLexer
323 else:
317 else:
324 pylexer = IPythonLexer
318 pylexer = IPythonLexer
325 tblexer = IPythonTracebackLexer
319 tblexer = IPythonTracebackLexer
326
320
327 self.pylexer = pylexer(**options)
321 self.pylexer = pylexer(**options)
328 self.tblexer = tblexer(**options)
322 self.tblexer = tblexer(**options)
329
323
330 self.reset()
324 self.reset()
331
325
332 def reset(self):
326 def reset(self):
333 self.mode = 'output'
327 self.mode = 'output'
334 self.index = 0
328 self.index = 0
335 self.buffer = u''
329 self.buffer = u''
336 self.insertions = []
330 self.insertions = []
337
331
338 def buffered_tokens(self):
332 def buffered_tokens(self):
339 """
333 """
340 Generator of unprocessed tokens after doing insertions and before
334 Generator of unprocessed tokens after doing insertions and before
341 changing to a new state.
335 changing to a new state.
342
336
343 """
337 """
344 if self.mode == 'output':
338 if self.mode == 'output':
345 tokens = [(0, Generic.Output, self.buffer)]
339 tokens = [(0, Generic.Output, self.buffer)]
346 elif self.mode == 'input':
340 elif self.mode == 'input':
347 tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
341 tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
348 else: # traceback
342 else: # traceback
349 tokens = self.tblexer.get_tokens_unprocessed(self.buffer)
343 tokens = self.tblexer.get_tokens_unprocessed(self.buffer)
350
344
351 for i, t, v in do_insertions(self.insertions, tokens):
345 for i, t, v in do_insertions(self.insertions, tokens):
352 # All token indexes are relative to the buffer.
346 # All token indexes are relative to the buffer.
353 yield self.index + i, t, v
347 yield self.index + i, t, v
354
348
355 # Clear it all
349 # Clear it all
356 self.index += len(self.buffer)
350 self.index += len(self.buffer)
357 self.buffer = u''
351 self.buffer = u''
358 self.insertions = []
352 self.insertions = []
359
353
360 def get_mci(self, line):
354 def get_mci(self, line):
361 """
355 """
362 Parses the line and returns a 3-tuple: (mode, code, insertion).
356 Parses the line and returns a 3-tuple: (mode, code, insertion).
363
357
364 `mode` is the next mode (or state) of the lexer, and is always equal
358 `mode` is the next mode (or state) of the lexer, and is always equal
365 to 'input', 'output', or 'tb'.
359 to 'input', 'output', or 'tb'.
366
360
367 `code` is a portion of the line that should be added to the buffer
361 `code` is a portion of the line that should be added to the buffer
368 corresponding to the next mode and eventually lexed by another lexer.
362 corresponding to the next mode and eventually lexed by another lexer.
369 For example, `code` could be Python code if `mode` were 'input'.
363 For example, `code` could be Python code if `mode` were 'input'.
370
364
371 `insertion` is a 3-tuple (index, token, text) representing an
365 `insertion` is a 3-tuple (index, token, text) representing an
372 unprocessed "token" that will be inserted into the stream of tokens
366 unprocessed "token" that will be inserted into the stream of tokens
373 that are created from the buffer once we change modes. This is usually
367 that are created from the buffer once we change modes. This is usually
374 the input or output prompt.
368 the input or output prompt.
375
369
376 In general, the next mode depends on current mode and on the contents
370 In general, the next mode depends on current mode and on the contents
377 of `line`.
371 of `line`.
378
372
379 """
373 """
380 # To reduce the number of regex match checks, we have multiple
374 # To reduce the number of regex match checks, we have multiple
381 # 'if' blocks instead of 'if-elif' blocks.
375 # 'if' blocks instead of 'if-elif' blocks.
382
376
383 # Check for possible end of input
377 # Check for possible end of input
384 in2_match = self.in2_regex.match(line)
378 in2_match = self.in2_regex.match(line)
385 in2_match_rstrip = self.in2_regex_rstrip.match(line)
379 in2_match_rstrip = self.in2_regex_rstrip.match(line)
386 if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
380 if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
387 in2_match_rstrip:
381 in2_match_rstrip:
388 end_input = True
382 end_input = True
389 else:
383 else:
390 end_input = False
384 end_input = False
391 if end_input and self.mode != 'tb':
385 if end_input and self.mode != 'tb':
392 # Only look for an end of input when not in tb mode.
386 # Only look for an end of input when not in tb mode.
393 # An ellipsis could appear within the traceback.
387 # An ellipsis could appear within the traceback.
394 mode = 'output'
388 mode = 'output'
395 code = u''
389 code = u''
396 insertion = (0, Generic.Prompt, line)
390 insertion = (0, Generic.Prompt, line)
397 return mode, code, insertion
391 return mode, code, insertion
398
392
399 # Check for output prompt
393 # Check for output prompt
400 out_match = self.out_regex.match(line)
394 out_match = self.out_regex.match(line)
401 out_match_rstrip = self.out_regex_rstrip.match(line)
395 out_match_rstrip = self.out_regex_rstrip.match(line)
402 if out_match or out_match_rstrip:
396 if out_match or out_match_rstrip:
403 mode = 'output'
397 mode = 'output'
404 if out_match:
398 if out_match:
405 idx = out_match.end()
399 idx = out_match.end()
406 else:
400 else:
407 idx = out_match_rstrip.end()
401 idx = out_match_rstrip.end()
408 code = line[idx:]
402 code = line[idx:]
409 # Use the 'heading' token for output. We cannot use Generic.Error
403 # Use the 'heading' token for output. We cannot use Generic.Error
410 # since it would conflict with exceptions.
404 # since it would conflict with exceptions.
411 insertion = (0, Generic.Heading, line[:idx])
405 insertion = (0, Generic.Heading, line[:idx])
412 return mode, code, insertion
406 return mode, code, insertion
413
407
414
408
415 # Check for input or continuation prompt (non stripped version)
409 # Check for input or continuation prompt (non stripped version)
416 in1_match = self.in1_regex.match(line)
410 in1_match = self.in1_regex.match(line)
417 if in1_match or (in2_match and self.mode != 'tb'):
411 if in1_match or (in2_match and self.mode != 'tb'):
418 # New input or when not in tb, continued input.
412 # New input or when not in tb, continued input.
419 # We do not check for continued input when in tb since it is
413 # We do not check for continued input when in tb since it is
420 # allowable to replace a long stack with an ellipsis.
414 # allowable to replace a long stack with an ellipsis.
421 mode = 'input'
415 mode = 'input'
422 if in1_match:
416 if in1_match:
423 idx = in1_match.end()
417 idx = in1_match.end()
424 else: # in2_match
418 else: # in2_match
425 idx = in2_match.end()
419 idx = in2_match.end()
426 code = line[idx:]
420 code = line[idx:]
427 insertion = (0, Generic.Prompt, line[:idx])
421 insertion = (0, Generic.Prompt, line[:idx])
428 return mode, code, insertion
422 return mode, code, insertion
429
423
430 # Check for input or continuation prompt (stripped version)
424 # Check for input or continuation prompt (stripped version)
431 in1_match_rstrip = self.in1_regex_rstrip.match(line)
425 in1_match_rstrip = self.in1_regex_rstrip.match(line)
432 if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
426 if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
433 # New input or when not in tb, continued input.
427 # New input or when not in tb, continued input.
434 # We do not check for continued input when in tb since it is
428 # We do not check for continued input when in tb since it is
435 # allowable to replace a long stack with an ellipsis.
429 # allowable to replace a long stack with an ellipsis.
436 mode = 'input'
430 mode = 'input'
437 if in1_match_rstrip:
431 if in1_match_rstrip:
438 idx = in1_match_rstrip.end()
432 idx = in1_match_rstrip.end()
439 else: # in2_match
433 else: # in2_match
440 idx = in2_match_rstrip.end()
434 idx = in2_match_rstrip.end()
441 code = line[idx:]
435 code = line[idx:]
442 insertion = (0, Generic.Prompt, line[:idx])
436 insertion = (0, Generic.Prompt, line[:idx])
443 return mode, code, insertion
437 return mode, code, insertion
444
438
445 # Check for traceback
439 # Check for traceback
446 if self.ipytb_start.match(line):
440 if self.ipytb_start.match(line):
447 mode = 'tb'
441 mode = 'tb'
448 code = line
442 code = line
449 insertion = None
443 insertion = None
450 return mode, code, insertion
444 return mode, code, insertion
451
445
452 # All other stuff...
446 # All other stuff...
453 if self.mode in ('input', 'output'):
447 if self.mode in ('input', 'output'):
454 # We assume all other text is output. Multiline input that
448 # We assume all other text is output. Multiline input that
455 # does not use the continuation marker cannot be detected.
449 # does not use the continuation marker cannot be detected.
456 # For example, the 3 in the following is clearly output:
450 # For example, the 3 in the following is clearly output:
457 #
451 #
458 # In [1]: print 3
452 # In [1]: print 3
459 # 3
453 # 3
460 #
454 #
461 # But the following second line is part of the input:
455 # But the following second line is part of the input:
462 #
456 #
463 # In [2]: while True:
457 # In [2]: while True:
464 # print True
458 # print True
465 #
459 #
466 # In both cases, the 2nd line will be 'output'.
460 # In both cases, the 2nd line will be 'output'.
467 #
461 #
468 mode = 'output'
462 mode = 'output'
469 else:
463 else:
470 mode = 'tb'
464 mode = 'tb'
471
465
472 code = line
466 code = line
473 insertion = None
467 insertion = None
474
468
475 return mode, code, insertion
469 return mode, code, insertion
476
470
477 def get_tokens_unprocessed(self, text):
471 def get_tokens_unprocessed(self, text):
478 self.reset()
472 self.reset()
479 for match in line_re.finditer(text):
473 for match in line_re.finditer(text):
480 line = match.group()
474 line = match.group()
481 mode, code, insertion = self.get_mci(line)
475 mode, code, insertion = self.get_mci(line)
482
476
483 if mode != self.mode:
477 if mode != self.mode:
484 # Yield buffered tokens before transitioning to new mode.
478 # Yield buffered tokens before transitioning to new mode.
485 for token in self.buffered_tokens():
479 for token in self.buffered_tokens():
486 yield token
480 yield token
487 self.mode = mode
481 self.mode = mode
488
482
489 if insertion:
483 if insertion:
490 self.insertions.append((len(self.buffer), [insertion]))
484 self.insertions.append((len(self.buffer), [insertion]))
491 self.buffer += code
485 self.buffer += code
492
486
493 for token in self.buffered_tokens():
487 for token in self.buffered_tokens():
494 yield token
488 yield token
495
489
496 class IPyLexer(Lexer):
490 class IPyLexer(Lexer):
497 r"""
491 r"""
498 Primary lexer for all IPython-like code.
492 Primary lexer for all IPython-like code.
499
493
500 This is a simple helper lexer. If the first line of the text begins with
494 This is a simple helper lexer. If the first line of the text begins with
501 "In \[[0-9]+\]:", then the entire text is parsed with an IPython console
495 "In \[[0-9]+\]:", then the entire text is parsed with an IPython console
502 lexer. If not, then the entire text is parsed with an IPython lexer.
496 lexer. If not, then the entire text is parsed with an IPython lexer.
503
497
504 The goal is to reduce the number of lexers that are registered
498 The goal is to reduce the number of lexers that are registered
505 with Pygments.
499 with Pygments.
506
500
507 """
501 """
508 name = 'IPy session'
502 name = 'IPy session'
509 aliases = ['ipy']
503 aliases = ['ipy']
510
504
511 def __init__(self, **options):
505 def __init__(self, **options):
512 self.python3 = get_bool_opt(options, 'python3', False)
506 self.python3 = get_bool_opt(options, 'python3', False)
513 if self.python3:
507 if self.python3:
514 self.aliases = ['ipy3']
508 self.aliases = ['ipy3']
515 else:
509 else:
516 self.aliases = ['ipy2', 'ipy']
510 self.aliases = ['ipy2', 'ipy']
517
511
518 Lexer.__init__(self, **options)
512 Lexer.__init__(self, **options)
519
513
520 self.IPythonLexer = IPythonLexer(**options)
514 self.IPythonLexer = IPythonLexer(**options)
521 self.IPythonConsoleLexer = IPythonConsoleLexer(**options)
515 self.IPythonConsoleLexer = IPythonConsoleLexer(**options)
522
516
523 def get_tokens_unprocessed(self, text):
517 def get_tokens_unprocessed(self, text):
524 # Search for the input prompt anywhere...this allows code blocks to
518 # Search for the input prompt anywhere...this allows code blocks to
525 # begin with comments as well.
519 # begin with comments as well.
526 if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL):
520 if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL):
527 lex = self.IPythonConsoleLexer
521 lex = self.IPythonConsoleLexer
528 else:
522 else:
529 lex = self.IPythonLexer
523 lex = self.IPythonLexer
530 for token in lex.get_tokens_unprocessed(text):
524 for token in lex.get_tokens_unprocessed(text):
531 yield token
525 yield token
532
526
@@ -1,114 +1,114 b''
1 """
1 """
2 Password generation for the IPython notebook.
2 Password generation for the IPython notebook.
3 """
3 """
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Imports
5 # Imports
6 #-----------------------------------------------------------------------------
6 #-----------------------------------------------------------------------------
7 # Stdlib
7 # Stdlib
8 import getpass
8 import getpass
9 import hashlib
9 import hashlib
10 import random
10 import random
11
11
12 # Our own
12 # Our own
13 from IPython.core.error import UsageError
13 from IPython.core.error import UsageError
14 from IPython.utils.py3compat import encode
14 from IPython.utils.py3compat import encode
15
15
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17 # Globals
17 # Globals
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19
19
20 # Length of the salt in nr of hex chars, which implies salt_len * 4
20 # Length of the salt in nr of hex chars, which implies salt_len * 4
21 # bits of randomness.
21 # bits of randomness.
22 salt_len = 12
22 salt_len = 12
23
23
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25 # Functions
25 # Functions
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27
27
28 def passwd(passphrase=None, algorithm='sha1'):
28 def passwd(passphrase=None, algorithm='sha1'):
29 """Generate hashed password and salt for use in notebook configuration.
29 """Generate hashed password and salt for use in notebook configuration.
30
30
31 In the notebook configuration, set `c.NotebookApp.password` to
31 In the notebook configuration, set `c.NotebookApp.password` to
32 the generated string.
32 the generated string.
33
33
34 Parameters
34 Parameters
35 ----------
35 ----------
36 passphrase : str
36 passphrase : str
37 Password to hash. If unspecified, the user is asked to input
37 Password to hash. If unspecified, the user is asked to input
38 and verify a password.
38 and verify a password.
39 algorithm : str
39 algorithm : str
40 Hashing algorithm to use (e.g, 'sha1' or any argument supported
40 Hashing algorithm to use (e.g, 'sha1' or any argument supported
41 by :func:`hashlib.new`).
41 by :func:`hashlib.new`).
42
42
43 Returns
43 Returns
44 -------
44 -------
45 hashed_passphrase : str
45 hashed_passphrase : str
46 Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'.
46 Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'.
47
47
48 Examples
48 Examples
49 --------
49 --------
50 >>> passwd('mypassword')
50 >>> passwd('mypassword')
51 'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12'
51 'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12' # random
52
52
53 """
53 """
54 if passphrase is None:
54 if passphrase is None:
55 for i in range(3):
55 for i in range(3):
56 p0 = getpass.getpass('Enter password: ')
56 p0 = getpass.getpass('Enter password: ')
57 p1 = getpass.getpass('Verify password: ')
57 p1 = getpass.getpass('Verify password: ')
58 if p0 == p1:
58 if p0 == p1:
59 passphrase = p0
59 passphrase = p0
60 break
60 break
61 else:
61 else:
62 print('Passwords do not match.')
62 print('Passwords do not match.')
63 else:
63 else:
64 raise UsageError('No matching passwords found. Giving up.')
64 raise UsageError('No matching passwords found. Giving up.')
65
65
66 h = hashlib.new(algorithm)
66 h = hashlib.new(algorithm)
67 salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len)
67 salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len)
68 h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii'))
68 h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii'))
69
69
70 return ':'.join((algorithm, salt, h.hexdigest()))
70 return ':'.join((algorithm, salt, h.hexdigest()))
71
71
72
72
73 def passwd_check(hashed_passphrase, passphrase):
73 def passwd_check(hashed_passphrase, passphrase):
74 """Verify that a given passphrase matches its hashed version.
74 """Verify that a given passphrase matches its hashed version.
75
75
76 Parameters
76 Parameters
77 ----------
77 ----------
78 hashed_passphrase : str
78 hashed_passphrase : str
79 Hashed password, in the format returned by `passwd`.
79 Hashed password, in the format returned by `passwd`.
80 passphrase : str
80 passphrase : str
81 Passphrase to validate.
81 Passphrase to validate.
82
82
83 Returns
83 Returns
84 -------
84 -------
85 valid : bool
85 valid : bool
86 True if the passphrase matches the hash.
86 True if the passphrase matches the hash.
87
87
88 Examples
88 Examples
89 --------
89 --------
90 >>> from IPython.lib.security import passwd_check
90 >>> from IPython.lib.security import passwd_check
91 >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
91 >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
92 ... 'mypassword')
92 ... 'mypassword')
93 True
93 True
94
94
95 >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
95 >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
96 ... 'anotherpassword')
96 ... 'anotherpassword')
97 False
97 False
98 """
98 """
99 try:
99 try:
100 algorithm, salt, pw_digest = hashed_passphrase.split(':', 2)
100 algorithm, salt, pw_digest = hashed_passphrase.split(':', 2)
101 except (ValueError, TypeError):
101 except (ValueError, TypeError):
102 return False
102 return False
103
103
104 try:
104 try:
105 h = hashlib.new(algorithm)
105 h = hashlib.new(algorithm)
106 except ValueError:
106 except ValueError:
107 return False
107 return False
108
108
109 if len(pw_digest) == 0:
109 if len(pw_digest) == 0:
110 return False
110 return False
111
111
112 h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii'))
112 h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii'))
113
113
114 return h.hexdigest() == pw_digest
114 return h.hexdigest() == pw_digest
@@ -1,157 +1,167 b''
1 """Simple example using doctests.
1 """Simple example using doctests.
2
2
3 This file just contains doctests both using plain python and IPython prompts.
3 This file just contains doctests both using plain python and IPython prompts.
4 All tests should be loaded by nose.
4 All tests should be loaded by nose.
5 """
5 """
6
6
7 import os
8
9
7 def pyfunc():
10 def pyfunc():
8 """Some pure python tests...
11 """Some pure python tests...
9
12
10 >>> pyfunc()
13 >>> pyfunc()
11 'pyfunc'
14 'pyfunc'
12
15
13 >>> import os
16 >>> import os
14
17
15 >>> 2+3
18 >>> 2+3
16 5
19 5
17
20
18 >>> for i in range(3):
21 >>> for i in range(3):
19 ... print(i, end=' ')
22 ... print(i, end=' ')
20 ... print(i+1, end=' ')
23 ... print(i+1, end=' ')
21 ...
24 ...
22 0 1 1 2 2 3
25 0 1 1 2 2 3
23 """
26 """
24 return 'pyfunc'
27 return 'pyfunc'
25
28
26 def ipfunc():
29 def ipfunc():
27 """Some ipython tests...
30 """Some ipython tests...
28
31
29 In [1]: import os
32 In [1]: import os
30
33
31 In [3]: 2+3
34 In [3]: 2+3
32 Out[3]: 5
35 Out[3]: 5
33
36
34 In [26]: for i in range(3):
37 In [26]: for i in range(3):
35 ....: print(i, end=' ')
38 ....: print(i, end=' ')
36 ....: print(i+1, end=' ')
39 ....: print(i+1, end=' ')
37 ....:
40 ....:
38 0 1 1 2 2 3
41 0 1 1 2 2 3
39
42
40
43
41 Examples that access the operating system work:
42
43 In [1]: !echo hello
44 hello
45
46 In [2]: !echo hello > /tmp/foo_iptest
47
48 In [3]: !cat /tmp/foo_iptest
49 hello
50
51 In [4]: rm -f /tmp/foo_iptest
52
53 It's OK to use '_' for the last result, but do NOT try to use IPython's
44 It's OK to use '_' for the last result, but do NOT try to use IPython's
54 numbered history of _NN outputs, since those won't exist under the
45 numbered history of _NN outputs, since those won't exist under the
55 doctest environment:
46 doctest environment:
56
47
57 In [7]: 'hi'
48 In [7]: 'hi'
58 Out[7]: 'hi'
49 Out[7]: 'hi'
59
50
60 In [8]: print(repr(_))
51 In [8]: print(repr(_))
61 'hi'
52 'hi'
62
53
63 In [7]: 3+4
54 In [7]: 3+4
64 Out[7]: 7
55 Out[7]: 7
65
56
66 In [8]: _+3
57 In [8]: _+3
67 Out[8]: 10
58 Out[8]: 10
68
59
69 In [9]: ipfunc()
60 In [9]: ipfunc()
70 Out[9]: 'ipfunc'
61 Out[9]: 'ipfunc'
71 """
62 """
72 return 'ipfunc'
63 return 'ipfunc'
73
64
74
65
66 def ipos():
67 """Examples that access the operating system work:
68
69 In [1]: !echo hello
70 hello
71
72 In [2]: !echo hello > /tmp/foo_iptest
73
74 In [3]: !cat /tmp/foo_iptest
75 hello
76
77 In [4]: rm -f /tmp/foo_iptest
78 """
79 pass
80
81
82 ipos.__skip_doctest__ = os.name == "nt"
83
84
75 def ranfunc():
85 def ranfunc():
76 """A function with some random output.
86 """A function with some random output.
77
87
78 Normal examples are verified as usual:
88 Normal examples are verified as usual:
79 >>> 1+3
89 >>> 1+3
80 4
90 4
81
91
82 But if you put '# random' in the output, it is ignored:
92 But if you put '# random' in the output, it is ignored:
83 >>> 1+3
93 >>> 1+3
84 junk goes here... # random
94 junk goes here... # random
85
95
86 >>> 1+2
96 >>> 1+2
87 again, anything goes #random
97 again, anything goes #random
88 if multiline, the random mark is only needed once.
98 if multiline, the random mark is only needed once.
89
99
90 >>> 1+2
100 >>> 1+2
91 You can also put the random marker at the end:
101 You can also put the random marker at the end:
92 # random
102 # random
93
103
94 >>> 1+2
104 >>> 1+2
95 # random
105 # random
96 .. or at the beginning.
106 .. or at the beginning.
97
107
98 More correct input is properly verified:
108 More correct input is properly verified:
99 >>> ranfunc()
109 >>> ranfunc()
100 'ranfunc'
110 'ranfunc'
101 """
111 """
102 return 'ranfunc'
112 return 'ranfunc'
103
113
104
114
105 def random_all():
115 def random_all():
106 """A function where we ignore the output of ALL examples.
116 """A function where we ignore the output of ALL examples.
107
117
108 Examples:
118 Examples:
109
119
110 # all-random
120 # all-random
111
121
112 This mark tells the testing machinery that all subsequent examples should
122 This mark tells the testing machinery that all subsequent examples should
113 be treated as random (ignoring their output). They are still executed,
123 be treated as random (ignoring their output). They are still executed,
114 so if a they raise an error, it will be detected as such, but their
124 so if a they raise an error, it will be detected as such, but their
115 output is completely ignored.
125 output is completely ignored.
116
126
117 >>> 1+3
127 >>> 1+3
118 junk goes here...
128 junk goes here...
119
129
120 >>> 1+3
130 >>> 1+3
121 klasdfj;
131 klasdfj;
122
132
123 >>> 1+2
133 >>> 1+2
124 again, anything goes
134 again, anything goes
125 blah...
135 blah...
126 """
136 """
127 pass
137 pass
128
138
129 def iprand():
139 def iprand():
130 """Some ipython tests with random output.
140 """Some ipython tests with random output.
131
141
132 In [7]: 3+4
142 In [7]: 3+4
133 Out[7]: 7
143 Out[7]: 7
134
144
135 In [8]: print('hello')
145 In [8]: print('hello')
136 world # random
146 world # random
137
147
138 In [9]: iprand()
148 In [9]: iprand()
139 Out[9]: 'iprand'
149 Out[9]: 'iprand'
140 """
150 """
141 return 'iprand'
151 return 'iprand'
142
152
143 def iprand_all():
153 def iprand_all():
144 """Some ipython tests with fully random output.
154 """Some ipython tests with fully random output.
145
155
146 # all-random
156 # all-random
147
157
148 In [7]: 1
158 In [7]: 1
149 Out[7]: 99
159 Out[7]: 99
150
160
151 In [8]: print('hello')
161 In [8]: print('hello')
152 world
162 world
153
163
154 In [9]: iprand_all()
164 In [9]: iprand_all()
155 Out[9]: 'junk'
165 Out[9]: 'junk'
156 """
166 """
157 return 'iprand_all'
167 return 'iprand_all'
@@ -1,30 +1,30 b''
1 =================================
1 =================================
2 Tests in example form - IPython
2 Tests in example form - IPython
3 =================================
3 =================================
4
4
5 You can write text files with examples that use IPython prompts (as long as you
5 You can write text files with examples that use IPython prompts (as long as you
6 use the nose ipython doctest plugin), but you can not mix and match prompt
6 use the nose ipython doctest plugin), but you can not mix and match prompt
7 styles in a single file. That is, you either use all ``>>>`` prompts or all
7 styles in a single file. That is, you either use all ``>>>`` prompts or all
8 IPython-style prompts. Your test suite *can* have both types, you just need to
8 IPython-style prompts. Your test suite *can* have both types, you just need to
9 put each type of example in a separate. Using IPython prompts, you can paste
9 put each type of example in a separate. Using IPython prompts, you can paste
10 directly from your session::
10 directly from your session::
11
11
12 In [5]: s="Hello World"
12 In [5]: s="Hello World"
13
13
14 In [6]: s.upper()
14 In [6]: s.upper()
15 Out[6]: 'HELLO WORLD'
15 Out[6]: 'HELLO WORLD'
16
16
17 Another example::
17 Another example::
18
18
19 In [8]: 1+3
19 In [8]: 1+3
20 Out[8]: 4
20 Out[8]: 4
21
21
22 Just like in IPython docstrings, you can use all IPython syntax and features::
22 Just like in IPython docstrings, you can use all IPython syntax and features::
23
23
24 In [9]: !echo "hello"
24 In [9]: !echo hello
25 hello
25 hello
26
26
27 In [10]: a='hi'
27 In [10]: a='hi'
28
28
29 In [11]: !echo $a
29 In [11]: !echo $a
30 hi
30 hi
@@ -1,164 +1,168 b''
1 """Tests for the decorators we've created for IPython.
1 """Tests for the decorators we've created for IPython.
2 """
2 """
3
3
4 # Module imports
4 # Module imports
5 # Std lib
5 # Std lib
6 import inspect
6 import inspect
7 import sys
7 import sys
8
8
9 # Third party
9 # Third party
10 import nose.tools as nt
10 import nose.tools as nt
11
11
12 # Our own
12 # Our own
13 from IPython.testing import decorators as dec
13 from IPython.testing import decorators as dec
14 from IPython.testing.skipdoctest import skip_doctest
14
15
15 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
16 # Utilities
17 # Utilities
17
18
18 # Note: copied from OInspect, kept here so the testing stuff doesn't create
19 # Note: copied from OInspect, kept here so the testing stuff doesn't create
19 # circular dependencies and is easier to reuse.
20 # circular dependencies and is easier to reuse.
20 def getargspec(obj):
21 def getargspec(obj):
21 """Get the names and default values of a function's arguments.
22 """Get the names and default values of a function's arguments.
22
23
23 A tuple of four things is returned: (args, varargs, varkw, defaults).
24 A tuple of four things is returned: (args, varargs, varkw, defaults).
24 'args' is a list of the argument names (it may contain nested lists).
25 'args' is a list of the argument names (it may contain nested lists).
25 'varargs' and 'varkw' are the names of the * and ** arguments or None.
26 'varargs' and 'varkw' are the names of the * and ** arguments or None.
26 'defaults' is an n-tuple of the default values of the last n arguments.
27 'defaults' is an n-tuple of the default values of the last n arguments.
27
28
28 Modified version of inspect.getargspec from the Python Standard
29 Modified version of inspect.getargspec from the Python Standard
29 Library."""
30 Library."""
30
31
31 if inspect.isfunction(obj):
32 if inspect.isfunction(obj):
32 func_obj = obj
33 func_obj = obj
33 elif inspect.ismethod(obj):
34 elif inspect.ismethod(obj):
34 func_obj = obj.__func__
35 func_obj = obj.__func__
35 else:
36 else:
36 raise TypeError('arg is not a Python function')
37 raise TypeError('arg is not a Python function')
37 args, varargs, varkw = inspect.getargs(func_obj.__code__)
38 args, varargs, varkw = inspect.getargs(func_obj.__code__)
38 return args, varargs, varkw, func_obj.__defaults__
39 return args, varargs, varkw, func_obj.__defaults__
39
40
40 #-----------------------------------------------------------------------------
41 #-----------------------------------------------------------------------------
41 # Testing functions
42 # Testing functions
42
43
43 @dec.as_unittest
44 @dec.as_unittest
44 def trivial():
45 def trivial():
45 """A trivial test"""
46 """A trivial test"""
46 pass
47 pass
47
48
48
49
49 @dec.skip()
50 @dec.skip()
50 def test_deliberately_broken():
51 def test_deliberately_broken():
51 """A deliberately broken test - we want to skip this one."""
52 """A deliberately broken test - we want to skip this one."""
52 1/0
53 1/0
53
54
54 @dec.skip('Testing the skip decorator')
55 @dec.skip('Testing the skip decorator')
55 def test_deliberately_broken2():
56 def test_deliberately_broken2():
56 """Another deliberately broken test - we want to skip this one."""
57 """Another deliberately broken test - we want to skip this one."""
57 1/0
58 1/0
58
59
59
60
60 # Verify that we can correctly skip the doctest for a function at will, but
61 # Verify that we can correctly skip the doctest for a function at will, but
61 # that the docstring itself is NOT destroyed by the decorator.
62 # that the docstring itself is NOT destroyed by the decorator.
63 @skip_doctest
62 def doctest_bad(x,y=1,**k):
64 def doctest_bad(x,y=1,**k):
63 """A function whose doctest we need to skip.
65 """A function whose doctest we need to skip.
64
66
65 >>> 1+1
67 >>> 1+1
66 3
68 3
67 """
69 """
68 print('x:',x)
70 print('x:',x)
69 print('y:',y)
71 print('y:',y)
70 print('k:',k)
72 print('k:',k)
71
73
72
74
73 def call_doctest_bad():
75 def call_doctest_bad():
74 """Check that we can still call the decorated functions.
76 """Check that we can still call the decorated functions.
75
77
76 >>> doctest_bad(3,y=4)
78 >>> doctest_bad(3,y=4)
77 x: 3
79 x: 3
78 y: 4
80 y: 4
79 k: {}
81 k: {}
80 """
82 """
81 pass
83 pass
82
84
83
85
84 def test_skip_dt_decorator():
86 def test_skip_dt_decorator():
85 """Doctest-skipping decorator should preserve the docstring.
87 """Doctest-skipping decorator should preserve the docstring.
86 """
88 """
87 # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
89 # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
88 check = """A function whose doctest we need to skip.
90 check = """A function whose doctest we need to skip.
89
91
90 >>> 1+1
92 >>> 1+1
91 3
93 3
92 """
94 """
93 # Fetch the docstring from doctest_bad after decoration.
95 # Fetch the docstring from doctest_bad after decoration.
94 val = doctest_bad.__doc__
96 val = doctest_bad.__doc__
95
97
96 nt.assert_equal(check,val,"doctest_bad docstrings don't match")
98 nt.assert_equal(check,val,"doctest_bad docstrings don't match")
97
99
98
100
99 # Doctest skipping should work for class methods too
101 # Doctest skipping should work for class methods too
100 class FooClass(object):
102 class FooClass(object):
101 """FooClass
103 """FooClass
102
104
103 Example:
105 Example:
104
106
105 >>> 1+1
107 >>> 1+1
106 2
108 2
107 """
109 """
108
110
111 @skip_doctest
109 def __init__(self,x):
112 def __init__(self,x):
110 """Make a FooClass.
113 """Make a FooClass.
111
114
112 Example:
115 Example:
113
116
114 >>> f = FooClass(3)
117 >>> f = FooClass(3)
115 junk
118 junk
116 """
119 """
117 print('Making a FooClass.')
120 print('Making a FooClass.')
118 self.x = x
121 self.x = x
119
122
123 @skip_doctest
120 def bar(self,y):
124 def bar(self,y):
121 """Example:
125 """Example:
122
126
123 >>> ff = FooClass(3)
127 >>> ff = FooClass(3)
124 >>> ff.bar(0)
128 >>> ff.bar(0)
125 boom!
129 boom!
126 >>> 1/0
130 >>> 1/0
127 bam!
131 bam!
128 """
132 """
129 return 1/y
133 return 1/y
130
134
131 def baz(self,y):
135 def baz(self,y):
132 """Example:
136 """Example:
133
137
134 >>> ff2 = FooClass(3)
138 >>> ff2 = FooClass(3)
135 Making a FooClass.
139 Making a FooClass.
136 >>> ff2.baz(3)
140 >>> ff2.baz(3)
137 True
141 True
138 """
142 """
139 return self.x==y
143 return self.x==y
140
144
141
145
142 def test_skip_dt_decorator2():
146 def test_skip_dt_decorator2():
143 """Doctest-skipping decorator should preserve function signature.
147 """Doctest-skipping decorator should preserve function signature.
144 """
148 """
145 # Hardcoded correct answer
149 # Hardcoded correct answer
146 dtargs = (['x', 'y'], None, 'k', (1,))
150 dtargs = (['x', 'y'], None, 'k', (1,))
147 # Introspect out the value
151 # Introspect out the value
148 dtargsr = getargspec(doctest_bad)
152 dtargsr = getargspec(doctest_bad)
149 assert dtargsr==dtargs, \
153 assert dtargsr==dtargs, \
150 "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
154 "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
151
155
152
156
153 @dec.skip_linux
157 @dec.skip_linux
154 def test_linux():
158 def test_linux():
155 nt.assert_false(sys.platform.startswith('linux'),"This test can't run under linux")
159 nt.assert_false(sys.platform.startswith('linux'),"This test can't run under linux")
156
160
157 @dec.skip_win32
161 @dec.skip_win32
158 def test_win32():
162 def test_win32():
159 nt.assert_not_equal(sys.platform,'win32',"This test can't run under windows")
163 nt.assert_not_equal(sys.platform,'win32',"This test can't run under windows")
160
164
161 @dec.skip_osx
165 @dec.skip_osx
162 def test_osx():
166 def test_osx():
163 nt.assert_not_equal(sys.platform,'darwin',"This test can't run under osx")
167 nt.assert_not_equal(sys.platform,'darwin',"This test can't run under osx")
164
168
@@ -1,440 +1,440 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 Utilities for path handling.
3 Utilities for path handling.
4 """
4 """
5
5
6 # Copyright (c) IPython Development Team.
6 # Copyright (c) IPython Development Team.
7 # Distributed under the terms of the Modified BSD License.
7 # Distributed under the terms of the Modified BSD License.
8
8
9 import os
9 import os
10 import sys
10 import sys
11 import errno
11 import errno
12 import shutil
12 import shutil
13 import random
13 import random
14 import glob
14 import glob
15 from warnings import warn
15 from warnings import warn
16
16
17 from IPython.utils.process import system
17 from IPython.utils.process import system
18 from IPython.utils.decorators import undoc
18 from IPython.utils.decorators import undoc
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Code
21 # Code
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23 fs_encoding = sys.getfilesystemencoding()
23 fs_encoding = sys.getfilesystemencoding()
24
24
25 def _writable_dir(path):
25 def _writable_dir(path):
26 """Whether `path` is a directory, to which the user has write access."""
26 """Whether `path` is a directory, to which the user has write access."""
27 return os.path.isdir(path) and os.access(path, os.W_OK)
27 return os.path.isdir(path) and os.access(path, os.W_OK)
28
28
29 if sys.platform == 'win32':
29 if sys.platform == 'win32':
30 def _get_long_path_name(path):
30 def _get_long_path_name(path):
31 """Get a long path name (expand ~) on Windows using ctypes.
31 """Get a long path name (expand ~) on Windows using ctypes.
32
32
33 Examples
33 Examples
34 --------
34 --------
35
35
36 >>> get_long_path_name('c:\\docume~1')
36 >>> get_long_path_name('c:\\\\docume~1')
37 'c:\\\\Documents and Settings'
37 'c:\\\\Documents and Settings'
38
38
39 """
39 """
40 try:
40 try:
41 import ctypes
41 import ctypes
42 except ImportError as e:
42 except ImportError as e:
43 raise ImportError('you need to have ctypes installed for this to work') from e
43 raise ImportError('you need to have ctypes installed for this to work') from e
44 _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
44 _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
45 _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
45 _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
46 ctypes.c_uint ]
46 ctypes.c_uint ]
47
47
48 buf = ctypes.create_unicode_buffer(260)
48 buf = ctypes.create_unicode_buffer(260)
49 rv = _GetLongPathName(path, buf, 260)
49 rv = _GetLongPathName(path, buf, 260)
50 if rv == 0 or rv > 260:
50 if rv == 0 or rv > 260:
51 return path
51 return path
52 else:
52 else:
53 return buf.value
53 return buf.value
54 else:
54 else:
55 def _get_long_path_name(path):
55 def _get_long_path_name(path):
56 """Dummy no-op."""
56 """Dummy no-op."""
57 return path
57 return path
58
58
59
59
60
60
61 def get_long_path_name(path):
61 def get_long_path_name(path):
62 """Expand a path into its long form.
62 """Expand a path into its long form.
63
63
64 On Windows this expands any ~ in the paths. On other platforms, it is
64 On Windows this expands any ~ in the paths. On other platforms, it is
65 a null operation.
65 a null operation.
66 """
66 """
67 return _get_long_path_name(path)
67 return _get_long_path_name(path)
68
68
69
69
70 def unquote_filename(name, win32=(sys.platform=='win32')):
70 def unquote_filename(name, win32=(sys.platform=='win32')):
71 """ On Windows, remove leading and trailing quotes from filenames.
71 """ On Windows, remove leading and trailing quotes from filenames.
72
72
73 This function has been deprecated and should not be used any more:
73 This function has been deprecated and should not be used any more:
74 unquoting is now taken care of by :func:`IPython.utils.process.arg_split`.
74 unquoting is now taken care of by :func:`IPython.utils.process.arg_split`.
75 """
75 """
76 warn("'unquote_filename' is deprecated since IPython 5.0 and should not "
76 warn("'unquote_filename' is deprecated since IPython 5.0 and should not "
77 "be used anymore", DeprecationWarning, stacklevel=2)
77 "be used anymore", DeprecationWarning, stacklevel=2)
78 if win32:
78 if win32:
79 if name.startswith(("'", '"')) and name.endswith(("'", '"')):
79 if name.startswith(("'", '"')) and name.endswith(("'", '"')):
80 name = name[1:-1]
80 name = name[1:-1]
81 return name
81 return name
82
82
83
83
84 def compress_user(path):
84 def compress_user(path):
85 """Reverse of :func:`os.path.expanduser`
85 """Reverse of :func:`os.path.expanduser`
86 """
86 """
87 home = os.path.expanduser('~')
87 home = os.path.expanduser('~')
88 if path.startswith(home):
88 if path.startswith(home):
89 path = "~" + path[len(home):]
89 path = "~" + path[len(home):]
90 return path
90 return path
91
91
92 def get_py_filename(name, force_win32=None):
92 def get_py_filename(name, force_win32=None):
93 """Return a valid python filename in the current directory.
93 """Return a valid python filename in the current directory.
94
94
95 If the given name is not a file, it adds '.py' and searches again.
95 If the given name is not a file, it adds '.py' and searches again.
96 Raises IOError with an informative message if the file isn't found.
96 Raises IOError with an informative message if the file isn't found.
97 """
97 """
98
98
99 name = os.path.expanduser(name)
99 name = os.path.expanduser(name)
100 if force_win32 is not None:
100 if force_win32 is not None:
101 warn("The 'force_win32' argument to 'get_py_filename' is deprecated "
101 warn("The 'force_win32' argument to 'get_py_filename' is deprecated "
102 "since IPython 5.0 and should not be used anymore",
102 "since IPython 5.0 and should not be used anymore",
103 DeprecationWarning, stacklevel=2)
103 DeprecationWarning, stacklevel=2)
104 if not os.path.isfile(name) and not name.endswith('.py'):
104 if not os.path.isfile(name) and not name.endswith('.py'):
105 name += '.py'
105 name += '.py'
106 if os.path.isfile(name):
106 if os.path.isfile(name):
107 return name
107 return name
108 else:
108 else:
109 raise IOError('File `%r` not found.' % name)
109 raise IOError('File `%r` not found.' % name)
110
110
111
111
112 def filefind(filename: str, path_dirs=None) -> str:
112 def filefind(filename: str, path_dirs=None) -> str:
113 """Find a file by looking through a sequence of paths.
113 """Find a file by looking through a sequence of paths.
114
114
115 This iterates through a sequence of paths looking for a file and returns
115 This iterates through a sequence of paths looking for a file and returns
116 the full, absolute path of the first occurrence of the file. If no set of
116 the full, absolute path of the first occurrence of the file. If no set of
117 path dirs is given, the filename is tested as is, after running through
117 path dirs is given, the filename is tested as is, after running through
118 :func:`expandvars` and :func:`expanduser`. Thus a simple call::
118 :func:`expandvars` and :func:`expanduser`. Thus a simple call::
119
119
120 filefind('myfile.txt')
120 filefind('myfile.txt')
121
121
122 will find the file in the current working dir, but::
122 will find the file in the current working dir, but::
123
123
124 filefind('~/myfile.txt')
124 filefind('~/myfile.txt')
125
125
126 Will find the file in the users home directory. This function does not
126 Will find the file in the users home directory. This function does not
127 automatically try any paths, such as the cwd or the user's home directory.
127 automatically try any paths, such as the cwd or the user's home directory.
128
128
129 Parameters
129 Parameters
130 ----------
130 ----------
131 filename : str
131 filename : str
132 The filename to look for.
132 The filename to look for.
133 path_dirs : str, None or sequence of str
133 path_dirs : str, None or sequence of str
134 The sequence of paths to look for the file in. If None, the filename
134 The sequence of paths to look for the file in. If None, the filename
135 need to be absolute or be in the cwd. If a string, the string is
135 need to be absolute or be in the cwd. If a string, the string is
136 put into a sequence and the searched. If a sequence, walk through
136 put into a sequence and the searched. If a sequence, walk through
137 each element and join with ``filename``, calling :func:`expandvars`
137 each element and join with ``filename``, calling :func:`expandvars`
138 and :func:`expanduser` before testing for existence.
138 and :func:`expanduser` before testing for existence.
139
139
140 Returns
140 Returns
141 -------
141 -------
142 path : str
142 path : str
143 returns absolute path to file.
143 returns absolute path to file.
144
144
145 Raises
145 Raises
146 ------
146 ------
147 IOError
147 IOError
148 """
148 """
149
149
150 # If paths are quoted, abspath gets confused, strip them...
150 # If paths are quoted, abspath gets confused, strip them...
151 filename = filename.strip('"').strip("'")
151 filename = filename.strip('"').strip("'")
152 # If the input is an absolute path, just check it exists
152 # If the input is an absolute path, just check it exists
153 if os.path.isabs(filename) and os.path.isfile(filename):
153 if os.path.isabs(filename) and os.path.isfile(filename):
154 return filename
154 return filename
155
155
156 if path_dirs is None:
156 if path_dirs is None:
157 path_dirs = ("",)
157 path_dirs = ("",)
158 elif isinstance(path_dirs, str):
158 elif isinstance(path_dirs, str):
159 path_dirs = (path_dirs,)
159 path_dirs = (path_dirs,)
160
160
161 for path in path_dirs:
161 for path in path_dirs:
162 if path == '.': path = os.getcwd()
162 if path == '.': path = os.getcwd()
163 testname = expand_path(os.path.join(path, filename))
163 testname = expand_path(os.path.join(path, filename))
164 if os.path.isfile(testname):
164 if os.path.isfile(testname):
165 return os.path.abspath(testname)
165 return os.path.abspath(testname)
166
166
167 raise IOError("File %r does not exist in any of the search paths: %r" %
167 raise IOError("File %r does not exist in any of the search paths: %r" %
168 (filename, path_dirs) )
168 (filename, path_dirs) )
169
169
170
170
171 class HomeDirError(Exception):
171 class HomeDirError(Exception):
172 pass
172 pass
173
173
174
174
175 def get_home_dir(require_writable=False) -> str:
175 def get_home_dir(require_writable=False) -> str:
176 """Return the 'home' directory, as a unicode string.
176 """Return the 'home' directory, as a unicode string.
177
177
178 Uses os.path.expanduser('~'), and checks for writability.
178 Uses os.path.expanduser('~'), and checks for writability.
179
179
180 See stdlib docs for how this is determined.
180 See stdlib docs for how this is determined.
181 For Python <3.8, $HOME is first priority on *ALL* platforms.
181 For Python <3.8, $HOME is first priority on *ALL* platforms.
182 For Python >=3.8 on Windows, %HOME% is no longer considered.
182 For Python >=3.8 on Windows, %HOME% is no longer considered.
183
183
184 Parameters
184 Parameters
185 ----------
185 ----------
186 require_writable : bool [default: False]
186 require_writable : bool [default: False]
187 if True:
187 if True:
188 guarantees the return value is a writable directory, otherwise
188 guarantees the return value is a writable directory, otherwise
189 raises HomeDirError
189 raises HomeDirError
190 if False:
190 if False:
191 The path is resolved, but it is not guaranteed to exist or be writable.
191 The path is resolved, but it is not guaranteed to exist or be writable.
192 """
192 """
193
193
194 homedir = os.path.expanduser('~')
194 homedir = os.path.expanduser('~')
195 # Next line will make things work even when /home/ is a symlink to
195 # Next line will make things work even when /home/ is a symlink to
196 # /usr/home as it is on FreeBSD, for example
196 # /usr/home as it is on FreeBSD, for example
197 homedir = os.path.realpath(homedir)
197 homedir = os.path.realpath(homedir)
198
198
199 if not _writable_dir(homedir) and os.name == 'nt':
199 if not _writable_dir(homedir) and os.name == 'nt':
200 # expanduser failed, use the registry to get the 'My Documents' folder.
200 # expanduser failed, use the registry to get the 'My Documents' folder.
201 try:
201 try:
202 import winreg as wreg
202 import winreg as wreg
203 with wreg.OpenKey(
203 with wreg.OpenKey(
204 wreg.HKEY_CURRENT_USER,
204 wreg.HKEY_CURRENT_USER,
205 r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
205 r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
206 ) as key:
206 ) as key:
207 homedir = wreg.QueryValueEx(key,'Personal')[0]
207 homedir = wreg.QueryValueEx(key,'Personal')[0]
208 except:
208 except:
209 pass
209 pass
210
210
211 if (not require_writable) or _writable_dir(homedir):
211 if (not require_writable) or _writable_dir(homedir):
212 assert isinstance(homedir, str), "Homedir should be unicode not bytes"
212 assert isinstance(homedir, str), "Homedir should be unicode not bytes"
213 return homedir
213 return homedir
214 else:
214 else:
215 raise HomeDirError('%s is not a writable dir, '
215 raise HomeDirError('%s is not a writable dir, '
216 'set $HOME environment variable to override' % homedir)
216 'set $HOME environment variable to override' % homedir)
217
217
218 def get_xdg_dir():
218 def get_xdg_dir():
219 """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
219 """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
220
220
221 This is only for non-OS X posix (Linux,Unix,etc.) systems.
221 This is only for non-OS X posix (Linux,Unix,etc.) systems.
222 """
222 """
223
223
224 env = os.environ
224 env = os.environ
225
225
226 if os.name == 'posix' and sys.platform != 'darwin':
226 if os.name == 'posix' and sys.platform != 'darwin':
227 # Linux, Unix, AIX, etc.
227 # Linux, Unix, AIX, etc.
228 # use ~/.config if empty OR not set
228 # use ~/.config if empty OR not set
229 xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
229 xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
230 if xdg and _writable_dir(xdg):
230 if xdg and _writable_dir(xdg):
231 assert isinstance(xdg, str)
231 assert isinstance(xdg, str)
232 return xdg
232 return xdg
233
233
234 return None
234 return None
235
235
236
236
237 def get_xdg_cache_dir():
237 def get_xdg_cache_dir():
238 """Return the XDG_CACHE_HOME, if it is defined and exists, else None.
238 """Return the XDG_CACHE_HOME, if it is defined and exists, else None.
239
239
240 This is only for non-OS X posix (Linux,Unix,etc.) systems.
240 This is only for non-OS X posix (Linux,Unix,etc.) systems.
241 """
241 """
242
242
243 env = os.environ
243 env = os.environ
244
244
245 if os.name == 'posix' and sys.platform != 'darwin':
245 if os.name == 'posix' and sys.platform != 'darwin':
246 # Linux, Unix, AIX, etc.
246 # Linux, Unix, AIX, etc.
247 # use ~/.cache if empty OR not set
247 # use ~/.cache if empty OR not set
248 xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
248 xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
249 if xdg and _writable_dir(xdg):
249 if xdg and _writable_dir(xdg):
250 assert isinstance(xdg, str)
250 assert isinstance(xdg, str)
251 return xdg
251 return xdg
252
252
253 return None
253 return None
254
254
255
255
256 @undoc
256 @undoc
257 def get_ipython_dir():
257 def get_ipython_dir():
258 warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
258 warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
259 from IPython.paths import get_ipython_dir
259 from IPython.paths import get_ipython_dir
260 return get_ipython_dir()
260 return get_ipython_dir()
261
261
262 @undoc
262 @undoc
263 def get_ipython_cache_dir():
263 def get_ipython_cache_dir():
264 warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
264 warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
265 from IPython.paths import get_ipython_cache_dir
265 from IPython.paths import get_ipython_cache_dir
266 return get_ipython_cache_dir()
266 return get_ipython_cache_dir()
267
267
268 @undoc
268 @undoc
269 def get_ipython_package_dir():
269 def get_ipython_package_dir():
270 warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
270 warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
271 from IPython.paths import get_ipython_package_dir
271 from IPython.paths import get_ipython_package_dir
272 return get_ipython_package_dir()
272 return get_ipython_package_dir()
273
273
274 @undoc
274 @undoc
275 def get_ipython_module_path(module_str):
275 def get_ipython_module_path(module_str):
276 warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
276 warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
277 from IPython.paths import get_ipython_module_path
277 from IPython.paths import get_ipython_module_path
278 return get_ipython_module_path(module_str)
278 return get_ipython_module_path(module_str)
279
279
280 @undoc
280 @undoc
281 def locate_profile(profile='default'):
281 def locate_profile(profile='default'):
282 warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
282 warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
283 from IPython.paths import locate_profile
283 from IPython.paths import locate_profile
284 return locate_profile(profile=profile)
284 return locate_profile(profile=profile)
285
285
286 def expand_path(s):
286 def expand_path(s):
287 """Expand $VARS and ~names in a string, like a shell
287 """Expand $VARS and ~names in a string, like a shell
288
288
289 :Examples:
289 :Examples:
290
290
291 In [2]: os.environ['FOO']='test'
291 In [2]: os.environ['FOO']='test'
292
292
293 In [3]: expand_path('variable FOO is $FOO')
293 In [3]: expand_path('variable FOO is $FOO')
294 Out[3]: 'variable FOO is test'
294 Out[3]: 'variable FOO is test'
295 """
295 """
296 # This is a pretty subtle hack. When expand user is given a UNC path
296 # This is a pretty subtle hack. When expand user is given a UNC path
297 # on Windows (\\server\share$\%username%), os.path.expandvars, removes
297 # on Windows (\\server\share$\%username%), os.path.expandvars, removes
298 # the $ to get (\\server\share\%username%). I think it considered $
298 # the $ to get (\\server\share\%username%). I think it considered $
299 # alone an empty var. But, we need the $ to remains there (it indicates
299 # alone an empty var. But, we need the $ to remains there (it indicates
300 # a hidden share).
300 # a hidden share).
301 if os.name=='nt':
301 if os.name=='nt':
302 s = s.replace('$\\', 'IPYTHON_TEMP')
302 s = s.replace('$\\', 'IPYTHON_TEMP')
303 s = os.path.expandvars(os.path.expanduser(s))
303 s = os.path.expandvars(os.path.expanduser(s))
304 if os.name=='nt':
304 if os.name=='nt':
305 s = s.replace('IPYTHON_TEMP', '$\\')
305 s = s.replace('IPYTHON_TEMP', '$\\')
306 return s
306 return s
307
307
308
308
309 def unescape_glob(string):
309 def unescape_glob(string):
310 """Unescape glob pattern in `string`."""
310 """Unescape glob pattern in `string`."""
311 def unescape(s):
311 def unescape(s):
312 for pattern in '*[]!?':
312 for pattern in '*[]!?':
313 s = s.replace(r'\{0}'.format(pattern), pattern)
313 s = s.replace(r'\{0}'.format(pattern), pattern)
314 return s
314 return s
315 return '\\'.join(map(unescape, string.split('\\\\')))
315 return '\\'.join(map(unescape, string.split('\\\\')))
316
316
317
317
318 def shellglob(args):
318 def shellglob(args):
319 """
319 """
320 Do glob expansion for each element in `args` and return a flattened list.
320 Do glob expansion for each element in `args` and return a flattened list.
321
321
322 Unmatched glob pattern will remain as-is in the returned list.
322 Unmatched glob pattern will remain as-is in the returned list.
323
323
324 """
324 """
325 expanded = []
325 expanded = []
326 # Do not unescape backslash in Windows as it is interpreted as
326 # Do not unescape backslash in Windows as it is interpreted as
327 # path separator:
327 # path separator:
328 unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
328 unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
329 for a in args:
329 for a in args:
330 expanded.extend(glob.glob(a) or [unescape(a)])
330 expanded.extend(glob.glob(a) or [unescape(a)])
331 return expanded
331 return expanded
332
332
333
333
334 def target_outdated(target,deps):
334 def target_outdated(target,deps):
335 """Determine whether a target is out of date.
335 """Determine whether a target is out of date.
336
336
337 target_outdated(target,deps) -> 1/0
337 target_outdated(target,deps) -> 1/0
338
338
339 deps: list of filenames which MUST exist.
339 deps: list of filenames which MUST exist.
340 target: single filename which may or may not exist.
340 target: single filename which may or may not exist.
341
341
342 If target doesn't exist or is older than any file listed in deps, return
342 If target doesn't exist or is older than any file listed in deps, return
343 true, otherwise return false.
343 true, otherwise return false.
344 """
344 """
345 try:
345 try:
346 target_time = os.path.getmtime(target)
346 target_time = os.path.getmtime(target)
347 except os.error:
347 except os.error:
348 return 1
348 return 1
349 for dep in deps:
349 for dep in deps:
350 dep_time = os.path.getmtime(dep)
350 dep_time = os.path.getmtime(dep)
351 if dep_time > target_time:
351 if dep_time > target_time:
352 #print "For target",target,"Dep failed:",dep # dbg
352 #print "For target",target,"Dep failed:",dep # dbg
353 #print "times (dep,tar):",dep_time,target_time # dbg
353 #print "times (dep,tar):",dep_time,target_time # dbg
354 return 1
354 return 1
355 return 0
355 return 0
356
356
357
357
358 def target_update(target,deps,cmd):
358 def target_update(target,deps,cmd):
359 """Update a target with a given command given a list of dependencies.
359 """Update a target with a given command given a list of dependencies.
360
360
361 target_update(target,deps,cmd) -> runs cmd if target is outdated.
361 target_update(target,deps,cmd) -> runs cmd if target is outdated.
362
362
363 This is just a wrapper around target_outdated() which calls the given
363 This is just a wrapper around target_outdated() which calls the given
364 command if target is outdated."""
364 command if target is outdated."""
365
365
366 if target_outdated(target,deps):
366 if target_outdated(target,deps):
367 system(cmd)
367 system(cmd)
368
368
369
369
370 ENOLINK = 1998
370 ENOLINK = 1998
371
371
372 def link(src, dst):
372 def link(src, dst):
373 """Hard links ``src`` to ``dst``, returning 0 or errno.
373 """Hard links ``src`` to ``dst``, returning 0 or errno.
374
374
375 Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
375 Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
376 supported by the operating system.
376 supported by the operating system.
377 """
377 """
378
378
379 if not hasattr(os, "link"):
379 if not hasattr(os, "link"):
380 return ENOLINK
380 return ENOLINK
381 link_errno = 0
381 link_errno = 0
382 try:
382 try:
383 os.link(src, dst)
383 os.link(src, dst)
384 except OSError as e:
384 except OSError as e:
385 link_errno = e.errno
385 link_errno = e.errno
386 return link_errno
386 return link_errno
387
387
388
388
389 def link_or_copy(src, dst):
389 def link_or_copy(src, dst):
390 """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.
390 """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.
391
391
392 Attempts to maintain the semantics of ``shutil.copy``.
392 Attempts to maintain the semantics of ``shutil.copy``.
393
393
394 Because ``os.link`` does not overwrite files, a unique temporary file
394 Because ``os.link`` does not overwrite files, a unique temporary file
395 will be used if the target already exists, then that file will be moved
395 will be used if the target already exists, then that file will be moved
396 into place.
396 into place.
397 """
397 """
398
398
399 if os.path.isdir(dst):
399 if os.path.isdir(dst):
400 dst = os.path.join(dst, os.path.basename(src))
400 dst = os.path.join(dst, os.path.basename(src))
401
401
402 link_errno = link(src, dst)
402 link_errno = link(src, dst)
403 if link_errno == errno.EEXIST:
403 if link_errno == errno.EEXIST:
404 if os.stat(src).st_ino == os.stat(dst).st_ino:
404 if os.stat(src).st_ino == os.stat(dst).st_ino:
405 # dst is already a hard link to the correct file, so we don't need
405 # dst is already a hard link to the correct file, so we don't need
406 # to do anything else. If we try to link and rename the file
406 # to do anything else. If we try to link and rename the file
407 # anyway, we get duplicate files - see http://bugs.python.org/issue21876
407 # anyway, we get duplicate files - see http://bugs.python.org/issue21876
408 return
408 return
409
409
410 new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
410 new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
411 try:
411 try:
412 link_or_copy(src, new_dst)
412 link_or_copy(src, new_dst)
413 except:
413 except:
414 try:
414 try:
415 os.remove(new_dst)
415 os.remove(new_dst)
416 except OSError:
416 except OSError:
417 pass
417 pass
418 raise
418 raise
419 os.rename(new_dst, dst)
419 os.rename(new_dst, dst)
420 elif link_errno != 0:
420 elif link_errno != 0:
421 # Either link isn't supported, or the filesystem doesn't support
421 # Either link isn't supported, or the filesystem doesn't support
422 # linking, or 'src' and 'dst' are on different filesystems.
422 # linking, or 'src' and 'dst' are on different filesystems.
423 shutil.copy(src, dst)
423 shutil.copy(src, dst)
424
424
425 def ensure_dir_exists(path, mode=0o755):
425 def ensure_dir_exists(path, mode=0o755):
426 """ensure that a directory exists
426 """ensure that a directory exists
427
427
428 If it doesn't exist, try to create it and protect against a race condition
428 If it doesn't exist, try to create it and protect against a race condition
429 if another process is doing the same.
429 if another process is doing the same.
430
430
431 The default permissions are 755, which differ from os.makedirs default of 777.
431 The default permissions are 755, which differ from os.makedirs default of 777.
432 """
432 """
433 if not os.path.exists(path):
433 if not os.path.exists(path):
434 try:
434 try:
435 os.makedirs(path, mode=mode)
435 os.makedirs(path, mode=mode)
436 except OSError as e:
436 except OSError as e:
437 if e.errno != errno.EEXIST:
437 if e.errno != errno.EEXIST:
438 raise
438 raise
439 elif not os.path.isdir(path):
439 elif not os.path.isdir(path):
440 raise IOError("%r exists but is not a directory" % path)
440 raise IOError("%r exists but is not a directory" % path)
General Comments 0
You need to be logged in to leave comments. Login now