##// END OF EJS Templates
unicode-related fixes in rwbase, nbformat tests
MinRK -
Show More
@@ -1,165 +1,174 b''
1 """Base classes and utilities for readers and writers.
1 """Base classes and utilities for readers and writers.
2
2
3 Authors:
3 Authors:
4
4
5 * Brian Granger
5 * Brian Granger
6 """
6 """
7
7
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9 # Copyright (C) 2008-2011 The IPython Development Team
9 # Copyright (C) 2008-2011 The IPython Development Team
10 #
10 #
11 # Distributed under the terms of the BSD License. The full license is in
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
12 # the file COPYING, distributed as part of this software.
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18
18
19 from base64 import encodestring, decodestring
19 from base64 import encodestring, decodestring
20 import pprint
20 import pprint
21
21
22 from IPython.utils.py3compat import str_to_bytes
22 from IPython.utils import py3compat
23
24 str_to_bytes = py3compat.str_to_bytes
23
25
24 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
25 # Code
27 # Code
26 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
27
29
28 def restore_bytes(nb):
30 def restore_bytes(nb):
29 """Restore bytes of image data from unicode-only formats.
31 """Restore bytes of image data from unicode-only formats.
30
32
31 Base64 encoding is handled elsewhere. Bytes objects in the notebook are
33 Base64 encoding is handled elsewhere. Bytes objects in the notebook are
32 always b64-encoded. We DO NOT encode/decode around file formats.
34 always b64-encoded. We DO NOT encode/decode around file formats.
33 """
35 """
34 for ws in nb.worksheets:
36 for ws in nb.worksheets:
35 for cell in ws.cells:
37 for cell in ws.cells:
36 if cell.cell_type == 'code':
38 if cell.cell_type == 'code':
37 for output in cell.outputs:
39 for output in cell.outputs:
38 if 'png' in output:
40 if 'png' in output:
39 output.png = str_to_bytes(output.png, 'ascii')
41 output.png = str_to_bytes(output.png, 'ascii')
40 if 'jpeg' in output:
42 if 'jpeg' in output:
41 output.jpeg = str_to_bytes(output.jpeg, 'ascii')
43 output.jpeg = str_to_bytes(output.jpeg, 'ascii')
42 return nb
44 return nb
43
45
44 # output keys that are likely to have multiline values
46 # output keys that are likely to have multiline values
45 _multiline_outputs = ['text', 'html', 'svg', 'latex', 'javascript', 'json']
47 _multiline_outputs = ['text', 'html', 'svg', 'latex', 'javascript', 'json']
46
48
47 def rejoin_lines(nb):
49 def rejoin_lines(nb):
48 """rejoin multiline text into strings
50 """rejoin multiline text into strings
49
51
50 For reversing effects of ``split_lines(nb)``.
52 For reversing effects of ``split_lines(nb)``.
51
53
52 This only rejoins lines that have been split, so if text objects were not split
54 This only rejoins lines that have been split, so if text objects were not split
53 they will pass through unchanged.
55 they will pass through unchanged.
54
56
55 Used when reading JSON files that may have been passed through split_lines.
57 Used when reading JSON files that may have been passed through split_lines.
56 """
58 """
57 for ws in nb.worksheets:
59 for ws in nb.worksheets:
58 for cell in ws.cells:
60 for cell in ws.cells:
59 if cell.cell_type == 'code':
61 if cell.cell_type == 'code':
60 if 'input' in cell and isinstance(cell.input, list):
62 if 'input' in cell and isinstance(cell.input, list):
61 cell.input = u'\n'.join(cell.input)
63 cell.input = u'\n'.join(cell.input)
62 for output in cell.outputs:
64 for output in cell.outputs:
63 for key in _multiline_outputs:
65 for key in _multiline_outputs:
64 item = output.get(key, None)
66 item = output.get(key, None)
65 if isinstance(item, list):
67 if isinstance(item, list):
66 output[key] = u'\n'.join(item)
68 output[key] = u'\n'.join(item)
67 else: # text, heading cell
69 else: # text, heading cell
68 for key in ['source', 'rendered']:
70 for key in ['source', 'rendered']:
69 item = cell.get(key, None)
71 item = cell.get(key, None)
70 if isinstance(item, list):
72 if isinstance(item, list):
71 cell[key] = u'\n'.join(item)
73 cell[key] = u'\n'.join(item)
72 return nb
74 return nb
73
75
74
76
75 def split_lines(nb):
77 def split_lines(nb):
76 """split likely multiline text into lists of strings
78 """split likely multiline text into lists of strings
77
79
78 For file output more friendly to line-based VCS. ``rejoin_lines(nb)`` will
80 For file output more friendly to line-based VCS. ``rejoin_lines(nb)`` will
79 reverse the effects of ``split_lines(nb)``.
81 reverse the effects of ``split_lines(nb)``.
80
82
81 Used when writing JSON files.
83 Used when writing JSON files.
82 """
84 """
83 for ws in nb.worksheets:
85 for ws in nb.worksheets:
84 for cell in ws.cells:
86 for cell in ws.cells:
85 if cell.cell_type == 'code':
87 if cell.cell_type == 'code':
86 if 'input' in cell and isinstance(cell.input, basestring):
88 if 'input' in cell and isinstance(cell.input, basestring):
87 cell.input = cell.input.splitlines()
89 cell.input = cell.input.splitlines()
88 for output in cell.outputs:
90 for output in cell.outputs:
89 for key in _multiline_outputs:
91 for key in _multiline_outputs:
90 item = output.get(key, None)
92 item = output.get(key, None)
91 if isinstance(item, basestring):
93 if isinstance(item, basestring):
92 output[key] = item.splitlines()
94 output[key] = item.splitlines()
93 else: # text, heading cell
95 else: # text, heading cell
94 for key in ['source', 'rendered']:
96 for key in ['source', 'rendered']:
95 item = cell.get(key, None)
97 item = cell.get(key, None)
96 if isinstance(item, basestring):
98 if isinstance(item, basestring):
97 cell[key] = item.splitlines()
99 cell[key] = item.splitlines()
98 return nb
100 return nb
99
101
100 # b64 encode/decode are never actually used, because all bytes objects in
102 # b64 encode/decode are never actually used, because all bytes objects in
101 # the notebook are already b64-encoded, and we don't need/want to double-encode
103 # the notebook are already b64-encoded, and we don't need/want to double-encode
102
104
103 def base64_decode(nb):
105 def base64_decode(nb):
104 """Restore all bytes objects in the notebook from base64-encoded strings.
106 """Restore all bytes objects in the notebook from base64-encoded strings.
105
107
106 Note: This is never used
108 Note: This is never used
107 """
109 """
108 for ws in nb.worksheets:
110 for ws in nb.worksheets:
109 for cell in ws.cells:
111 for cell in ws.cells:
110 if cell.cell_type == 'code':
112 if cell.cell_type == 'code':
111 for output in cell.outputs:
113 for output in cell.outputs:
112 if 'png' in output:
114 if 'png' in output:
113 if isinstance(output.png, unicode):
115 if isinstance(output.png, unicode):
114 output.png = output.png.encode('ascii')
116 output.png = output.png.encode('ascii')
115 output.png = decodestring(output.png)
117 output.png = decodestring(output.png)
116 if 'jpeg' in output:
118 if 'jpeg' in output:
117 if isinstance(output.jpeg, unicode):
119 if isinstance(output.jpeg, unicode):
118 output.jpeg = output.jpeg.encode('ascii')
120 output.jpeg = output.jpeg.encode('ascii')
119 output.jpeg = decodestring(output.jpeg)
121 output.jpeg = decodestring(output.jpeg)
120 return nb
122 return nb
121
123
122
124
123 def base64_encode(nb):
125 def base64_encode(nb):
124 """Base64 encode all bytes objects in the notebook.
126 """Base64 encode all bytes objects in the notebook.
125
127
126 These will be b64-encoded unicode strings
128 These will be b64-encoded unicode strings
127
129
128 Note: This is never used
130 Note: This is never used
129 """
131 """
130 for ws in nb.worksheets:
132 for ws in nb.worksheets:
131 for cell in ws.cells:
133 for cell in ws.cells:
132 if cell.cell_type == 'code':
134 if cell.cell_type == 'code':
133 for output in cell.outputs:
135 for output in cell.outputs:
134 if 'png' in output:
136 if 'png' in output:
135 output.png = encodestring(output.png).decode('ascii')
137 output.png = encodestring(output.png).decode('ascii')
136 if 'jpeg' in output:
138 if 'jpeg' in output:
137 output.jpeg = encodestring(output.jpeg).decode('ascii')
139 output.jpeg = encodestring(output.jpeg).decode('ascii')
138 return nb
140 return nb
139
141
140
142
141 class NotebookReader(object):
143 class NotebookReader(object):
142 """A class for reading notebooks."""
144 """A class for reading notebooks."""
143
145
144 def reads(self, s, **kwargs):
146 def reads(self, s, **kwargs):
145 """Read a notebook from a string."""
147 """Read a notebook from a string."""
146 raise NotImplementedError("loads must be implemented in a subclass")
148 raise NotImplementedError("loads must be implemented in a subclass")
147
149
148 def read(self, fp, **kwargs):
150 def read(self, fp, **kwargs):
149 """Read a notebook from a file like object"""
151 """Read a notebook from a file like object"""
150 return self.read(fp.read(), **kwargs)
152 nbs = fp.read()
153 if not py3compat.PY3 and not isinstance(nbs, unicode):
154 nbs = py3compat.str_to_unicode(nbs)
155 return self.reads(nbs, **kwargs)
151
156
152
157
153 class NotebookWriter(object):
158 class NotebookWriter(object):
154 """A class for writing notebooks."""
159 """A class for writing notebooks."""
155
160
156 def writes(self, nb, **kwargs):
161 def writes(self, nb, **kwargs):
157 """Write a notebook to a string."""
162 """Write a notebook to a string."""
158 raise NotImplementedError("loads must be implemented in a subclass")
163 raise NotImplementedError("loads must be implemented in a subclass")
159
164
160 def write(self, nb, fp, **kwargs):
165 def write(self, nb, fp, **kwargs):
161 """Write a notebook to a file like object"""
166 """Write a notebook to a file like object"""
162 return fp.write(self.writes(nb,**kwargs))
167 nbs = self.writes(nb,**kwargs)
168 if not py3compat.PY3 and not isinstance(nbs, unicode):
169 # this branch is likely only taken for JSON on Python 2
170 nbs = py3compat.str_to_unicode(nbs)
171 return fp.write(nbs)
163
172
164
173
165
174
@@ -1,127 +1,129 b''
1 # -*- coding: utf-8 -*-
2
1 import os
3 import os
2 from base64 import encodestring
4 from base64 import encodestring
3
5
4 from ..nbbase import (
6 from ..nbbase import (
5 NotebookNode,
7 NotebookNode,
6 new_code_cell, new_text_cell, new_worksheet, new_notebook, new_output,
8 new_code_cell, new_text_cell, new_worksheet, new_notebook, new_output,
7 new_metadata, new_author, new_heading_cell, nbformat
9 new_metadata, new_author, new_heading_cell, nbformat
8 )
10 )
9
11
10 # some random base64-encoded *bytes*
12 # some random base64-encoded *bytes*
11 png = encodestring(os.urandom(5))
13 png = encodestring(os.urandom(5))
12 jpeg = encodestring(os.urandom(6))
14 jpeg = encodestring(os.urandom(6))
13
15
14 ws = new_worksheet(name='worksheet1')
16 ws = new_worksheet(name='worksheet1')
15
17
16 ws.cells.append(new_text_cell(
18 ws.cells.append(new_text_cell(
17 u'html',
19 u'html',
18 source='Some NumPy Examples',
20 source='Some NumPy Examples',
19 rendered='Some NumPy Examples'
21 rendered='Some NumPy Examples'
20 ))
22 ))
21
23
22
24
23 ws.cells.append(new_code_cell(
25 ws.cells.append(new_code_cell(
24 input='import numpy',
26 input='import numpy',
25 prompt_number=1,
27 prompt_number=1,
26 collapsed=False
28 collapsed=False
27 ))
29 ))
28
30
29 ws.cells.append(new_text_cell(
31 ws.cells.append(new_text_cell(
30 u'markdown',
32 u'markdown',
31 source='A random array',
33 source='A random array',
32 rendered='A random array'
34 rendered='A random array'
33 ))
35 ))
34
36
35 ws.cells.append(new_text_cell(
37 ws.cells.append(new_text_cell(
36 u'plaintext',
38 u'plaintext',
37 source='A random array',
39 source='A random array',
38 ))
40 ))
39
41
40 ws.cells.append(new_heading_cell(
42 ws.cells.append(new_heading_cell(
41 u'My Heading',
43 u'My Heading',
42 level=2
44 level=2
43 ))
45 ))
44
46
45 ws.cells.append(new_code_cell(
47 ws.cells.append(new_code_cell(
46 input='a = numpy.random.rand(100)',
48 input='a = numpy.random.rand(100)',
47 prompt_number=2,
49 prompt_number=2,
48 collapsed=True
50 collapsed=True
49 ))
51 ))
50
52
51 ws.cells.append(new_code_cell(
53 ws.cells.append(new_code_cell(
52 input='print a',
54 input=u'print "ünîcødé"',
53 prompt_number=3,
55 prompt_number=3,
54 collapsed=False,
56 collapsed=False,
55 outputs=[new_output(
57 outputs=[new_output(
56 output_type=u'pyout',
58 output_type=u'pyout',
57 output_text=u'<array a>',
59 output_text=u'<array a>',
58 output_html=u'The HTML rep',
60 output_html=u'The HTML rep',
59 output_latex=u'$a$',
61 output_latex=u'$a$',
60 output_png=png,
62 output_png=png,
61 output_jpeg=jpeg,
63 output_jpeg=jpeg,
62 output_svg=u'<svg>',
64 output_svg=u'<svg>',
63 output_json=u'json data',
65 output_json=u'json data',
64 output_javascript=u'var i=0;',
66 output_javascript=u'var i=0;',
65 prompt_number=3
67 prompt_number=3
66 ),new_output(
68 ),new_output(
67 output_type=u'display_data',
69 output_type=u'display_data',
68 output_text=u'<array a>',
70 output_text=u'<array a>',
69 output_html=u'The HTML rep',
71 output_html=u'The HTML rep',
70 output_latex=u'$a$',
72 output_latex=u'$a$',
71 output_png=png,
73 output_png=png,
72 output_jpeg=jpeg,
74 output_jpeg=jpeg,
73 output_svg=u'<svg>',
75 output_svg=u'<svg>',
74 output_json=u'json data',
76 output_json=u'json data',
75 output_javascript=u'var i=0;'
77 output_javascript=u'var i=0;'
76 ),new_output(
78 ),new_output(
77 output_type=u'pyerr',
79 output_type=u'pyerr',
78 etype=u'NameError',
80 etype=u'NameError',
79 evalue=u'NameError was here',
81 evalue=u'NameError was here',
80 traceback=[u'frame 0', u'frame 1', u'frame 2']
82 traceback=[u'frame 0', u'frame 1', u'frame 2']
81 )]
83 )]
82 ))
84 ))
83
85
84 authors = [new_author(name='Bart Simpson',email='bsimpson@fox.com',
86 authors = [new_author(name='Bart Simpson',email='bsimpson@fox.com',
85 affiliation=u'Fox',url=u'http://www.fox.com')]
87 affiliation=u'Fox',url=u'http://www.fox.com')]
86 md = new_metadata(name=u'My Notebook',license=u'BSD',created=u'8601_goes_here',
88 md = new_metadata(name=u'My Notebook',license=u'BSD',created=u'8601_goes_here',
87 modified=u'8601_goes_here',gistid=u'21341231',authors=authors)
89 modified=u'8601_goes_here',gistid=u'21341231',authors=authors)
88
90
89 nb0 = new_notebook(
91 nb0 = new_notebook(
90 worksheets=[ws, new_worksheet(name='worksheet2')],
92 worksheets=[ws, new_worksheet(name='worksheet2')],
91 metadata=md
93 metadata=md
92 )
94 )
93
95
94 nb0_py = """# -*- coding: utf-8 -*-
96 nb0_py = u"""# -*- coding: utf-8 -*-
95 # <nbformat>%i</nbformat>
97 # <nbformat>%i</nbformat>
96
98
97 # <htmlcell>
99 # <htmlcell>
98
100
99 # Some NumPy Examples
101 # Some NumPy Examples
100
102
101 # <codecell>
103 # <codecell>
102
104
103 import numpy
105 import numpy
104
106
105 # <markdowncell>
107 # <markdowncell>
106
108
107 # A random array
109 # A random array
108
110
109 # <plaintextcell>
111 # <plaintextcell>
110
112
111 # A random array
113 # A random array
112
114
113 # <headingcell level=2>
115 # <headingcell level=2>
114
116
115 # My Heading
117 # My Heading
116
118
117 # <codecell>
119 # <codecell>
118
120
119 a = numpy.random.rand(100)
121 a = numpy.random.rand(100)
120
122
121 # <codecell>
123 # <codecell>
122
124
123 print a
125 print "ünîcødé"
124
126
125 """ % nbformat
127 """ % nbformat
126
128
127
129
General Comments 0
You need to be logged in to leave comments. Login now