##// END OF EJS Templates
remove sys_version for Python 3...
Paul Ivanov -
r22959:7851f8a3
parent child
Show More
Add another comment
@@ -604,22 +604,21 class Pdb(OldPdb, object):
604 ('Globals', self.curframe.f_globals)]
604 ('Globals', self.curframe.f_globals)]
605 self.shell.find_line_magic('psource')(arg, namespaces=namespaces)
605 self.shell.find_line_magic('psource')(arg, namespaces=namespaces)
606
606
607 if sys.version_info > (3, ):
607 def do_where(self, arg):
608 def do_where(self, arg):
608 """w(here)
609 """w(here)
609 Print a stack trace, with the most recent frame at the bottom.
610 Print a stack trace, with the most recent frame at the bottom.
610 An arrow indicates the "current frame", which determines the
611 An arrow indicates the "current frame", which determines the
611 context of most commands. 'bt' is an alias for this command.
612 context of most commands. 'bt' is an alias for this command.
612
613
613 Take a number as argument as an (optional) number of context line to
614 Take a number as argument as an (optional) number of context line to
614 print"""
615 print"""
615 if arg:
616 if arg:
616 context = int(arg)
617 context = int(arg)
617 self.print_stack_trace(context)
618 self.print_stack_trace(context)
618 else:
619 else:
619 self.print_stack_trace()
620 self.print_stack_trace()
621
620
622 do_w = do_where
621 do_w = do_where
623
622
624
623
625 def set_trace(frame=None):
624 def set_trace(frame=None):
@@ -187,10 +187,7 class ExecutionResult(object):
187 raise self.error_in_exec
187 raise self.error_in_exec
188
188
189 def __repr__(self):
189 def __repr__(self):
190 if sys.version_info > (3,):
190 name = self.__class__.__qualname__
191 name = self.__class__.__qualname__
192 else:
193 name = self.__class__.__name__
194 return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s result=%s>' %\
191 return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s result=%s>' %\
195 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.result))
192 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.result))
196
193
@@ -155,7 +155,6 def test_latex_completions():
155
155
156
156
157
157
158 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
159 def test_back_latex_completion():
158 def test_back_latex_completion():
160 ip = get_ipython()
159 ip = get_ipython()
161
160
@@ -164,7 +163,6 def test_back_latex_completion():
164 nt.assert_equal(len(matches), 1)
163 nt.assert_equal(len(matches), 1)
165 nt.assert_equal(matches[0], '\\beta')
164 nt.assert_equal(matches[0], '\\beta')
166
165
167 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
168 def test_back_unicode_completion():
166 def test_back_unicode_completion():
169 ip = get_ipython()
167 ip = get_ipython()
170
168
@@ -173,7 +171,6 def test_back_unicode_completion():
173 nt.assert_equal(matches[0], '\\ROMAN NUMERAL FIVE')
171 nt.assert_equal(matches[0], '\\ROMAN NUMERAL FIVE')
174
172
175
173
176 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
177 def test_forward_unicode_completion():
174 def test_forward_unicode_completion():
178 ip = get_ipython()
175 ip = get_ipython()
179
176
@@ -181,7 +178,6 def test_forward_unicode_completion():
181 nt.assert_equal(len(matches), 1)
178 nt.assert_equal(len(matches), 1)
182 nt.assert_equal(matches[0], 'Ⅴ')
179 nt.assert_equal(matches[0], 'Ⅴ')
183
180
184 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
185 @dec.knownfailureif(sys.platform == 'win32', 'Fails if there is a C:\\j... path')
181 @dec.knownfailureif(sys.platform == 'win32', 'Fails if there is a C:\\j... path')
186 def test_no_ascii_back_completion():
182 def test_no_ascii_back_completion():
187 ip = get_ipython()
183 ip = get_ipython()
@@ -588,7 +584,6 def test_dict_key_completion_contexts():
588
584
589
585
590
586
591 @dec.onlyif(sys.version_info[0] >= 3, 'This test only applies in Py>=3')
592 def test_dict_key_completion_bytes():
587 def test_dict_key_completion_bytes():
593 """Test handling of bytes in dict key completion"""
588 """Test handling of bytes in dict key completion"""
594 ip = get_ipython()
589 ip = get_ipython()
@@ -618,7 +613,6 def test_dict_key_completion_bytes():
618 nt.assert_not_in("abd", matches)
613 nt.assert_not_in("abd", matches)
619
614
620
615
621 @dec.onlyif(sys.version_info[0] < 3, 'This test only applies in Py<3')
622 def test_dict_key_completion_unicode_py2():
616 def test_dict_key_completion_unicode_py2():
623 """Test handling of unicode in dict key completion"""
617 """Test handling of unicode in dict key completion"""
624 ip = get_ipython()
618 ip = get_ipython()
@@ -679,7 +673,6 def test_dict_key_completion_unicode_py2():
679 nt.assert_in(u"d[u'a\u05d0b']", matches)
673 nt.assert_in(u"d[u'a\u05d0b']", matches)
680
674
681
675
682 @dec.onlyif(sys.version_info[0] >= 3, 'This test only applies in Py>=3')
683 def test_dict_key_completion_unicode_py3():
676 def test_dict_key_completion_unicode_py3():
684 """Test handling of unicode in dict key completion"""
677 """Test handling of unicode in dict key completion"""
685 ip = get_ipython()
678 ip = get_ipython()
@@ -338,19 +338,6 class InteractiveShellTestCase(unittest.TestCase):
338 finally:
338 finally:
339 trap.hook = save_hook
339 trap.hook = save_hook
340
340
341 @skipif(sys.version_info[0] >= 3, "softspace removed in py3")
342 def test_print_softspace(self):
343 """Verify that softspace is handled correctly when executing multiple
344 statements.
345
346 In [1]: print 1; print 2
347 1
348 2
349
350 In [2]: print 1,; print 2
351 1 2
352 """
353
354 def test_ofind_line_magic(self):
341 def test_ofind_line_magic(self):
355 from IPython.core.magic import register_line_magic
342 from IPython.core.magic import register_line_magic
356
343
@@ -466,22 +453,6 class InteractiveShellTestCase(unittest.TestCase):
466 # Reset the custom exception hook
453 # Reset the custom exception hook
467 ip.set_custom_exc((), None)
454 ip.set_custom_exc((), None)
468
455
469 @skipif(sys.version_info[0] >= 3, "no differences with __future__ in py3")
470 def test_future_environment(self):
471 "Can we run code with & without the shell's __future__ imports?"
472 ip.run_cell("from __future__ import division")
473 ip.run_cell("a = 1/2", shell_futures=True)
474 self.assertEqual(ip.user_ns['a'], 0.5)
475 ip.run_cell("b = 1/2", shell_futures=False)
476 self.assertEqual(ip.user_ns['b'], 0)
477
478 ip.compile.reset_compiler_flags()
479 # This shouldn't leak to the shell's compiler
480 ip.run_cell("from __future__ import division \nc=1/2", shell_futures=False)
481 self.assertEqual(ip.user_ns['c'], 0.5)
482 ip.run_cell("d = 1/2", shell_futures=True)
483 self.assertEqual(ip.user_ns['d'], 0)
484
485 def test_mktempfile(self):
456 def test_mktempfile(self):
486 filename = ip.mktempfile()
457 filename = ip.mktempfile()
487 # Check that we can open the file again on Windows
458 # Check that we can open the file again on Windows
@@ -509,10 +480,7 class InteractiveShellTestCase(unittest.TestCase):
509 raise DerivedInterrupt("foo")
480 raise DerivedInterrupt("foo")
510 except KeyboardInterrupt:
481 except KeyboardInterrupt:
511 msg = ip.get_exception_only()
482 msg = ip.get_exception_only()
512 if sys.version_info[0] <= 2:
483 self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n')
513 self.assertEqual(msg, 'DerivedInterrupt: foo\n')
514 else:
515 self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n')
516
484
517 def test_inspect_text(self):
485 def test_inspect_text(self):
518 ip.run_cell('a = 5')
486 ip.run_cell('a = 5')
@@ -377,18 +377,6 def test_time3():
377 "run = 0\n"
377 "run = 0\n"
378 "run += 1")
378 "run += 1")
379
379
380 @dec.skipif(sys.version_info[0] >= 3, "no differences with __future__ in py3")
381 def test_time_futures():
382 "Test %time with __future__ environments"
383 ip = get_ipython()
384 ip.autocall = 0
385 ip.run_cell("from __future__ import division")
386 with tt.AssertPrints('0.25'):
387 ip.run_line_magic('time', 'print(1/4)')
388 ip.compile.reset_compiler_flags()
389 with tt.AssertNotPrints('0.25'):
390 ip.run_line_magic('time', 'print(1/4)')
391
392 def test_doctest_mode():
380 def test_doctest_mode():
393 "Toggle doctest_mode twice, it should be a no-op and run without error"
381 "Toggle doctest_mode twice, it should be a no-op and run without error"
394 _ip.magic('doctest_mode')
382 _ip.magic('doctest_mode')
@@ -573,17 +561,6 def test_timeit_return_quiet():
573 res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
561 res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
574 assert (res is not None)
562 assert (res is not None)
575
563
576 @dec.skipif(sys.version_info[0] >= 3, "no differences with __future__ in py3")
577 def test_timeit_futures():
578 "Test %timeit with __future__ environments"
579 ip = get_ipython()
580 ip.run_cell("from __future__ import division")
581 with tt.AssertPrints('0.25'):
582 ip.run_line_magic('timeit', '-n1 -r1 print(1/4)')
583 ip.compile.reset_compiler_flags()
584 with tt.AssertNotPrints('0.25'):
585 ip.run_line_magic('timeit', '-n1 -r1 print(1/4)')
586
587 @dec.skipif(execution.profile is None)
564 @dec.skipif(execution.profile is None)
588 def test_prun_special_syntax():
565 def test_prun_special_syntax():
589 "Test %%prun with IPython special syntax"
566 "Test %%prun with IPython special syntax"
@@ -57,8 +57,6 def check_cpaste(code, should_fail=False):
57 finally:
57 finally:
58 sys.stdin = stdin_save
58 sys.stdin = stdin_save
59
59
60 PY31 = sys.version_info[:2] == (3,1)
61
62 def test_cpaste():
60 def test_cpaste():
63 """Test cpaste magic"""
61 """Test cpaste magic"""
64
62
@@ -77,13 +75,8 def test_cpaste():
77 ],
75 ],
78
76
79 'fail': ["1 + runf()",
77 'fail': ["1 + runf()",
78 "++ runf()",
80 ]}
79 ]}
81
82 # I don't know why this is failing specifically on Python 3.1. I've
83 # checked it manually interactively, but we don't care enough about 3.1
84 # to spend time fiddling with the tests, so we just skip it.
85 if not PY31:
86 tests['fail'].append("++ runf()")
87
80
88 ip.user_ns['runf'] = runf
81 ip.user_ns['runf'] = runf
89
82
@@ -275,8 +275,7 def test_info():
275 nt.assert_equal(i['type_name'], 'type')
275 nt.assert_equal(i['type_name'], 'type')
276 expted_class = str(type(type)) # <class 'type'> (Python 3) or <type 'type'>
276 expted_class = str(type(type)) # <class 'type'> (Python 3) or <type 'type'>
277 nt.assert_equal(i['base_class'], expted_class)
277 nt.assert_equal(i['base_class'], expted_class)
278 if sys.version_info > (3,):
278 nt.assert_regex(i['string_form'], "<class 'IPython.core.tests.test_oinspect.Call'( at 0x[0-9a-f]{1,9})?>")
279 nt.assert_regex(i['string_form'], "<class 'IPython.core.tests.test_oinspect.Call'( at 0x[0-9a-f]{1,9})?>")
280 fname = __file__
279 fname = __file__
281 if fname.endswith(".pyc"):
280 if fname.endswith(".pyc"):
282 fname = fname[:-1]
281 fname = fname[:-1]
@@ -313,44 +313,43 def r3o2():
313 #----------------------------------------------------------------------------
313 #----------------------------------------------------------------------------
314
314
315 # module testing (minimal)
315 # module testing (minimal)
316 if sys.version_info > (3,):
316 def test_handlers():
317 def test_handlers():
317 def spam(c, d_e):
318 def spam(c, d_e):
318 (d, e) = d_e
319 (d, e) = d_e
319 x = c + d
320 x = c + d
320 y = c * d
321 y = c * d
321 foo(x, y)
322 foo(x, y)
322
323
323 def foo(a, b, bar=1):
324 def foo(a, b, bar=1):
324 eggs(a, b + bar)
325 eggs(a, b + bar)
325
326
326 def eggs(f, g, z=globals()):
327 def eggs(f, g, z=globals()):
327 h = f + g
328 h = f + g
328 i = f - g
329 i = f - g
329 return h / i
330 return h / i
330
331
331 buff = io.StringIO()
332 buff = io.StringIO()
332
333
333 buff.write('')
334 buff.write('')
334 buff.write('*** Before ***')
335 buff.write('*** Before ***')
335 try:
336 try:
336 buff.write(spam(1, (2, 3)))
337 buff.write(spam(1, (2, 3)))
337 except:
338 except:
338 traceback.print_exc(file=buff)
339 traceback.print_exc(file=buff)
339
340
340 handler = ColorTB(ostream=buff)
341 handler = ColorTB(ostream=buff)
341 buff.write('*** ColorTB ***')
342 buff.write('*** ColorTB ***')
342 try:
343 try:
343 buff.write(spam(1, (2, 3)))
344 buff.write(spam(1, (2, 3)))
344 except:
345 except:
345 handler(*sys.exc_info())
346 handler(*sys.exc_info())
346 buff.write('')
347 buff.write('')
347
348
348 handler = VerboseTB(ostream=buff)
349 handler = VerboseTB(ostream=buff)
349 buff.write('*** VerboseTB ***')
350 buff.write('*** VerboseTB ***')
350 try:
351 try:
351 buff.write(spam(1, (2, 3)))
352 buff.write(spam(1, (2, 3)))
352 except:
353 except:
353 handler(*sys.exc_info())
354 handler(*sys.exc_info())
354 buff.write('')
355 buff.write('')
356
355
@@ -45,8 +45,7 pjoin = path.join
45
45
46 # Enable printing all warnings raise by IPython's modules
46 # Enable printing all warnings raise by IPython's modules
47 warnings.filterwarnings('ignore', message='.*Matplotlib is building the font cache.*', category=UserWarning, module='.*')
47 warnings.filterwarnings('ignore', message='.*Matplotlib is building the font cache.*', category=UserWarning, module='.*')
48 if sys.version_info > (3,0):
48 warnings.filterwarnings('error', message='.*', category=ResourceWarning, module='.*')
49 warnings.filterwarnings('error', message='.*', category=ResourceWarning, module='.*')
50 warnings.filterwarnings('error', message=".*{'config': True}.*", category=DeprecationWarning, module='IPy.*')
49 warnings.filterwarnings('error', message=".*{'config': True}.*", category=DeprecationWarning, module='IPy.*')
51 warnings.filterwarnings('default', message='.*', category=Warning, module='IPy.*')
50 warnings.filterwarnings('default', message='.*', category=Warning, module='IPy.*')
52
51
@@ -1,26 +1,13
1 """Decorators marks that a doctest should be skipped, for both python 2 and 3.
1 """Decorators marks that a doctest should be skipped.
2
2
3 The IPython.testing.decorators module triggers various extra imports, including
3 The IPython.testing.decorators module triggers various extra imports, including
4 numpy and sympy if they're present. Since this decorator is used in core parts
4 numpy and sympy if they're present. Since this decorator is used in core parts
5 of IPython, it's in a separate module so that running IPython doesn't trigger
5 of IPython, it's in a separate module so that running IPython doesn't trigger
6 those imports."""
6 those imports."""
7
7
8 #-----------------------------------------------------------------------------
8 # Copyright (C) IPython Development Team
9 # Copyright (C) 2009-2011 The IPython Development Team
9 # Distributed under the terms of the Modified BSD License.
10 #
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
13 #-----------------------------------------------------------------------------
14
10
15 #-----------------------------------------------------------------------------
16 # Imports
17 #-----------------------------------------------------------------------------
18
19 import sys
20
21 #-----------------------------------------------------------------------------
22 # Decorators
23 #-----------------------------------------------------------------------------
24
11
25 def skip_doctest(f):
12 def skip_doctest(f):
26 """Decorator - mark a function or method for skipping its doctest.
13 """Decorator - mark a function or method for skipping its doctest.
This diff has been collapsed as it changes many lines, (594 lines changed) Show them Hide them
@@ -1,9 +1,595
1 """Load our patched versions of tokenize.
1 """Patched version of standard library tokenize, to deal with various bugs.
2
3 Based on Python 3.2 code.
4
5 Patches:
6
7 - Gareth Rees' patch for Python issue #12691 (untokenizing)
8 - Except we don't encode the output of untokenize
9 - Python 2 compatible syntax, so that it can be byte-compiled at installation
10 - Newlines in comments and blank lines should be either NL or NEWLINE, depending
11 on whether they are in a multi-line statement. Filed as Python issue #17061.
12 - Export generate_tokens & TokenError
13 - u and rb literals are allowed under Python 3.3 and above.
14
15 ------------------------------------------------------------------------------
16 Tokenization help for Python programs.
17
18 tokenize(readline) is a generator that breaks a stream of bytes into
19 Python tokens. It decodes the bytes according to PEP-0263 for
20 determining source file encoding.
21
22 It accepts a readline-like method which is called repeatedly to get the
23 next line of input (or b"" for EOF). It generates 5-tuples with these
24 members:
25
26 the token type (see token.py)
27 the token (a string)
28 the starting (row, column) indices of the token (a 2-tuple of ints)
29 the ending (row, column) indices of the token (a 2-tuple of ints)
30 the original line (string)
31
32 It is designed to match the working of the Python tokenizer exactly, except
33 that it produces COMMENT tokens for comments and gives type OP for all
34 operators. Additionally, all token lists start with an ENCODING token
35 which tells you which encoding was used to decode the bytes stream.
2 """
36 """
37 from __future__ import absolute_import
3
38
39 __author__ = 'Ka-Ping Yee <ping@lfw.org>'
40 __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
41 'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
42 'Michael Foord')
43 import builtins
44 import re
4 import sys
45 import sys
46 from token import *
47 from codecs import lookup, BOM_UTF8
48 import collections
49 from io import TextIOWrapper
50 cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
51
52 import token
53 __all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
54 "NL", "untokenize", "ENCODING", "TokenInfo"]
55 del token
56
57 __all__ += ["generate_tokens", "TokenError"]
5
58
6 if sys.version_info[0] >= 3:
59 COMMENT = N_TOKENS
7 from ._tokenize_py3 import *
60 tok_name[COMMENT] = 'COMMENT'
61 NL = N_TOKENS + 1
62 tok_name[NL] = 'NL'
63 ENCODING = N_TOKENS + 2
64 tok_name[ENCODING] = 'ENCODING'
65 N_TOKENS += 3
66
67 class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
68 def __repr__(self):
69 annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
70 return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
71 self._replace(type=annotated_type))
72
73 def group(*choices): return '(' + '|'.join(choices) + ')'
74 def any(*choices): return group(*choices) + '*'
75 def maybe(*choices): return group(*choices) + '?'
76
77 # Note: we use unicode matching for names ("\w") but ascii matching for
78 # number literals.
79 Whitespace = r'[ \f\t]*'
80 Comment = r'#[^\r\n]*'
81 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
82 Name = r'\w+'
83
84 Hexnumber = r'0[xX][0-9a-fA-F]+'
85 Binnumber = r'0[bB][01]+'
86 Octnumber = r'0[oO][0-7]+'
87 Decnumber = r'(?:0+|[1-9][0-9]*)'
88 Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
89 Exponent = r'[eE][-+]?[0-9]+'
90 Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
91 Expfloat = r'[0-9]+' + Exponent
92 Floatnumber = group(Pointfloat, Expfloat)
93 Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
94 Number = group(Imagnumber, Floatnumber, Intnumber)
95
96 if sys.version_info.minor >= 3:
97 StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?'
8 else:
98 else:
9 from ._tokenize_py2 import *
99 StringPrefix = r'(?:[bB]?[rR]?)?'
100
101 # Tail end of ' string.
102 Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
103 # Tail end of " string.
104 Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
105 # Tail end of ''' string.
106 Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
107 # Tail end of """ string.
108 Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
109 Triple = group(StringPrefix + "'''", StringPrefix + '"""')
110 # Single-line ' or " string.
111 String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
112 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
113
114 # Because of leftmost-then-longest match semantics, be sure to put the
115 # longest operators first (e.g., if = came before ==, == would get
116 # recognized as two instances of =).
117 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
118 r"//=?", r"->",
119 r"[+\-*/%&|^=<>]=?",
120 r"~")
121
122 Bracket = '[][(){}]'
123 Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
124 Funny = group(Operator, Bracket, Special)
125
126 PlainToken = group(Number, Funny, String, Name)
127 Token = Ignore + PlainToken
128
129 # First (or only) line of ' or " string.
130 ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
131 group("'", r'\\\r?\n'),
132 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
133 group('"', r'\\\r?\n'))
134 PseudoExtras = group(r'\\\r?\n', Comment, Triple)
135 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
136
137 def _compile(expr):
138 return re.compile(expr, re.UNICODE)
139
140 tokenprog, pseudoprog, single3prog, double3prog = map(
141 _compile, (Token, PseudoToken, Single3, Double3))
142 endprogs = {"'": _compile(Single), '"': _compile(Double),
143 "'''": single3prog, '"""': double3prog,
144 "r'''": single3prog, 'r"""': double3prog,
145 "b'''": single3prog, 'b"""': double3prog,
146 "R'''": single3prog, 'R"""': double3prog,
147 "B'''": single3prog, 'B"""': double3prog,
148 "br'''": single3prog, 'br"""': double3prog,
149 "bR'''": single3prog, 'bR"""': double3prog,
150 "Br'''": single3prog, 'Br"""': double3prog,
151 "BR'''": single3prog, 'BR"""': double3prog,
152 'r': None, 'R': None, 'b': None, 'B': None}
153
154 triple_quoted = {}
155 for t in ("'''", '"""',
156 "r'''", 'r"""', "R'''", 'R"""',
157 "b'''", 'b"""', "B'''", 'B"""',
158 "br'''", 'br"""', "Br'''", 'Br"""',
159 "bR'''", 'bR"""', "BR'''", 'BR"""'):
160 triple_quoted[t] = t
161 single_quoted = {}
162 for t in ("'", '"',
163 "r'", 'r"', "R'", 'R"',
164 "b'", 'b"', "B'", 'B"',
165 "br'", 'br"', "Br'", 'Br"',
166 "bR'", 'bR"', "BR'", 'BR"' ):
167 single_quoted[t] = t
168
169 if sys.version_info.minor >= 3:
170 # Python 3.3
171 for _prefix in ['rb', 'rB', 'Rb', 'RB', 'u', 'U']:
172 _t2 = _prefix+'"""'
173 endprogs[_t2] = double3prog
174 triple_quoted[_t2] = _t2
175 _t1 = _prefix + "'''"
176 endprogs[_t1] = single3prog
177 triple_quoted[_t1] = _t1
178 single_quoted[_prefix+'"'] = _prefix+'"'
179 single_quoted[_prefix+"'"] = _prefix+"'"
180 del _prefix, _t2, _t1
181 endprogs['u'] = None
182 endprogs['U'] = None
183
184 del _compile
185
186 tabsize = 8
187
188 class TokenError(Exception): pass
189
190 class StopTokenizing(Exception): pass
191
192
193 class Untokenizer:
194
195 def __init__(self):
196 self.tokens = []
197 self.prev_row = 1
198 self.prev_col = 0
199 self.encoding = 'utf-8'
200
201 def add_whitespace(self, tok_type, start):
202 row, col = start
203 assert row >= self.prev_row
204 col_offset = col - self.prev_col
205 if col_offset > 0:
206 self.tokens.append(" " * col_offset)
207 elif row > self.prev_row and tok_type not in (NEWLINE, NL, ENDMARKER):
208 # Line was backslash-continued.
209 self.tokens.append(" ")
210
211 def untokenize(self, tokens):
212 iterable = iter(tokens)
213 for t in iterable:
214 if len(t) == 2:
215 self.compat(t, iterable)
216 break
217 tok_type, token, start, end = t[:4]
218 if tok_type == ENCODING:
219 self.encoding = token
220 continue
221 self.add_whitespace(tok_type, start)
222 self.tokens.append(token)
223 self.prev_row, self.prev_col = end
224 if tok_type in (NEWLINE, NL):
225 self.prev_row += 1
226 self.prev_col = 0
227 return "".join(self.tokens)
228
229 def compat(self, token, iterable):
230 # This import is here to avoid problems when the itertools
231 # module is not built yet and tokenize is imported.
232 from itertools import chain
233 startline = False
234 prevstring = False
235 indents = []
236 toks_append = self.tokens.append
237
238 for tok in chain([token], iterable):
239 toknum, tokval = tok[:2]
240 if toknum == ENCODING:
241 self.encoding = tokval
242 continue
243
244 if toknum in (NAME, NUMBER):
245 tokval += ' '
246
247 # Insert a space between two consecutive strings
248 if toknum == STRING:
249 if prevstring:
250 tokval = ' ' + tokval
251 prevstring = True
252 else:
253 prevstring = False
254
255 if toknum == INDENT:
256 indents.append(tokval)
257 continue
258 elif toknum == DEDENT:
259 indents.pop()
260 continue
261 elif toknum in (NEWLINE, NL):
262 startline = True
263 elif startline and indents:
264 toks_append(indents[-1])
265 startline = False
266 toks_append(tokval)
267
268
269 def untokenize(tokens):
270 """
271 Convert ``tokens`` (an iterable) back into Python source code. Return
272 a bytes object, encoded using the encoding specified by the last
273 ENCODING token in ``tokens``, or UTF-8 if no ENCODING token is found.
274
275 The result is guaranteed to tokenize back to match the input so that
276 the conversion is lossless and round-trips are assured. The
277 guarantee applies only to the token type and token string as the
278 spacing between tokens (column positions) may change.
279
280 :func:`untokenize` has two modes. If the input tokens are sequences
281 of length 2 (``type``, ``string``) then spaces are added as necessary to
282 preserve the round-trip property.
283
284 If the input tokens are sequences of length 4 or more (``type``,
285 ``string``, ``start``, ``end``), as returned by :func:`tokenize`, then
286 spaces are added so that each token appears in the result at the
287 position indicated by ``start`` and ``end``, if possible.
288 """
289 return Untokenizer().untokenize(tokens)
290
291
292 def _get_normal_name(orig_enc):
293 """Imitates get_normal_name in tokenizer.c."""
294 # Only care about the first 12 characters.
295 enc = orig_enc[:12].lower().replace("_", "-")
296 if enc == "utf-8" or enc.startswith("utf-8-"):
297 return "utf-8"
298 if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
299 enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
300 return "iso-8859-1"
301 return orig_enc
302
303 def detect_encoding(readline):
304 """
305 The detect_encoding() function is used to detect the encoding that should
306 be used to decode a Python source file. It requires one argment, readline,
307 in the same way as the tokenize() generator.
308
309 It will call readline a maximum of twice, and return the encoding used
310 (as a string) and a list of any lines (left as bytes) it has read in.
311