##// END OF EJS Templates
catch `MemoryError` in compiling
yangyang -
Show More
@@ -1,173 +1,173 b''
1 """
1 """
2 Async helper function that are invalid syntax on Python 3.5 and below.
2 Async helper function that are invalid syntax on Python 3.5 and below.
3
3
4 This code is best effort, and may have edge cases not behaving as expected. In
4 This code is best effort, and may have edge cases not behaving as expected. In
5 particular it contain a number of heuristics to detect whether code is
5 particular it contain a number of heuristics to detect whether code is
6 effectively async and need to run in an event loop or not.
6 effectively async and need to run in an event loop or not.
7
7
8 Some constructs (like top-level `return`, or `yield`) are taken care of
8 Some constructs (like top-level `return`, or `yield`) are taken care of
9 explicitly to actually raise a SyntaxError and stay as close as possible to
9 explicitly to actually raise a SyntaxError and stay as close as possible to
10 Python semantics.
10 Python semantics.
11 """
11 """
12
12
13
13
14 import ast
14 import ast
15 import sys
15 import sys
16 import inspect
16 import inspect
17 from textwrap import dedent, indent
17 from textwrap import dedent, indent
18
18
19
19
20 class _AsyncIORunner:
20 class _AsyncIORunner:
21
21
22 def __call__(self, coro):
22 def __call__(self, coro):
23 """
23 """
24 Handler for asyncio autoawait
24 Handler for asyncio autoawait
25 """
25 """
26 import asyncio
26 import asyncio
27
27
28 return asyncio.get_event_loop().run_until_complete(coro)
28 return asyncio.get_event_loop().run_until_complete(coro)
29
29
30 def __str__(self):
30 def __str__(self):
31 return 'asyncio'
31 return 'asyncio'
32
32
33 _asyncio_runner = _AsyncIORunner()
33 _asyncio_runner = _AsyncIORunner()
34
34
35
35
36 def _curio_runner(coroutine):
36 def _curio_runner(coroutine):
37 """
37 """
38 handler for curio autoawait
38 handler for curio autoawait
39 """
39 """
40 import curio
40 import curio
41
41
42 return curio.run(coroutine)
42 return curio.run(coroutine)
43
43
44
44
45 def _trio_runner(async_fn):
45 def _trio_runner(async_fn):
46 import trio
46 import trio
47
47
48 async def loc(coro):
48 async def loc(coro):
49 """
49 """
50 We need the dummy no-op async def to protect from
50 We need the dummy no-op async def to protect from
51 trio's internal. See https://github.com/python-trio/trio/issues/89
51 trio's internal. See https://github.com/python-trio/trio/issues/89
52 """
52 """
53 return await coro
53 return await coro
54
54
55 return trio.run(loc, async_fn)
55 return trio.run(loc, async_fn)
56
56
57
57
58 def _pseudo_sync_runner(coro):
58 def _pseudo_sync_runner(coro):
59 """
59 """
60 A runner that does not really allow async execution, and just advance the coroutine.
60 A runner that does not really allow async execution, and just advance the coroutine.
61
61
62 See discussion in https://github.com/python-trio/trio/issues/608,
62 See discussion in https://github.com/python-trio/trio/issues/608,
63
63
64 Credit to Nathaniel Smith
64 Credit to Nathaniel Smith
65
65
66 """
66 """
67 try:
67 try:
68 coro.send(None)
68 coro.send(None)
69 except StopIteration as exc:
69 except StopIteration as exc:
70 return exc.value
70 return exc.value
71 else:
71 else:
72 # TODO: do not raise but return an execution result with the right info.
72 # TODO: do not raise but return an execution result with the right info.
73 raise RuntimeError(
73 raise RuntimeError(
74 "{coro_name!r} needs a real async loop".format(coro_name=coro.__name__)
74 "{coro_name!r} needs a real async loop".format(coro_name=coro.__name__)
75 )
75 )
76
76
77
77
78 def _asyncify(code: str) -> str:
78 def _asyncify(code: str) -> str:
79 """wrap code in async def definition.
79 """wrap code in async def definition.
80
80
81 And setup a bit of context to run it later.
81 And setup a bit of context to run it later.
82 """
82 """
83 res = dedent(
83 res = dedent(
84 """
84 """
85 async def __wrapper__():
85 async def __wrapper__():
86 try:
86 try:
87 {usercode}
87 {usercode}
88 finally:
88 finally:
89 locals()
89 locals()
90 """
90 """
91 ).format(usercode=indent(code, " " * 8))
91 ).format(usercode=indent(code, " " * 8))
92 return res
92 return res
93
93
94
94
95 class _AsyncSyntaxErrorVisitor(ast.NodeVisitor):
95 class _AsyncSyntaxErrorVisitor(ast.NodeVisitor):
96 """
96 """
97 Find syntax errors that would be an error in an async repl, but because
97 Find syntax errors that would be an error in an async repl, but because
98 the implementation involves wrapping the repl in an async function, it
98 the implementation involves wrapping the repl in an async function, it
99 is erroneously allowed (e.g. yield or return at the top level)
99 is erroneously allowed (e.g. yield or return at the top level)
100 """
100 """
101 def __init__(self):
101 def __init__(self):
102 if sys.version_info >= (3,8):
102 if sys.version_info >= (3,8):
103 raise ValueError('DEPRECATED in Python 3.8+')
103 raise ValueError('DEPRECATED in Python 3.8+')
104 self.depth = 0
104 self.depth = 0
105 super().__init__()
105 super().__init__()
106
106
107 def generic_visit(self, node):
107 def generic_visit(self, node):
108 func_types = (ast.FunctionDef, ast.AsyncFunctionDef)
108 func_types = (ast.FunctionDef, ast.AsyncFunctionDef)
109 invalid_types_by_depth = {
109 invalid_types_by_depth = {
110 0: (ast.Return, ast.Yield, ast.YieldFrom),
110 0: (ast.Return, ast.Yield, ast.YieldFrom),
111 1: (ast.Nonlocal,)
111 1: (ast.Nonlocal,)
112 }
112 }
113
113
114 should_traverse = self.depth < max(invalid_types_by_depth.keys())
114 should_traverse = self.depth < max(invalid_types_by_depth.keys())
115 if isinstance(node, func_types) and should_traverse:
115 if isinstance(node, func_types) and should_traverse:
116 self.depth += 1
116 self.depth += 1
117 super().generic_visit(node)
117 super().generic_visit(node)
118 self.depth -= 1
118 self.depth -= 1
119 elif isinstance(node, invalid_types_by_depth[self.depth]):
119 elif isinstance(node, invalid_types_by_depth[self.depth]):
120 raise SyntaxError()
120 raise SyntaxError()
121 else:
121 else:
122 super().generic_visit(node)
122 super().generic_visit(node)
123
123
124
124
125 def _async_parse_cell(cell: str) -> ast.AST:
125 def _async_parse_cell(cell: str) -> ast.AST:
126 """
126 """
127 This is a compatibility shim for pre-3.7 when async outside of a function
127 This is a compatibility shim for pre-3.7 when async outside of a function
128 is a syntax error at the parse stage.
128 is a syntax error at the parse stage.
129
129
130 It will return an abstract syntax tree parsed as if async and await outside
130 It will return an abstract syntax tree parsed as if async and await outside
131 of a function were not a syntax error.
131 of a function were not a syntax error.
132 """
132 """
133 if sys.version_info < (3, 7):
133 if sys.version_info < (3, 7):
134 # Prior to 3.7 you need to asyncify before parse
134 # Prior to 3.7 you need to asyncify before parse
135 wrapped_parse_tree = ast.parse(_asyncify(cell))
135 wrapped_parse_tree = ast.parse(_asyncify(cell))
136 return wrapped_parse_tree.body[0].body[0]
136 return wrapped_parse_tree.body[0].body[0]
137 else:
137 else:
138 return ast.parse(cell)
138 return ast.parse(cell)
139
139
140
140
141 def _should_be_async(cell: str) -> bool:
141 def _should_be_async(cell: str) -> bool:
142 """Detect if a block of code need to be wrapped in an `async def`
142 """Detect if a block of code need to be wrapped in an `async def`
143
143
144 Attempt to parse the block of code, it it compile we're fine.
144 Attempt to parse the block of code, it it compile we're fine.
145 Otherwise we wrap if and try to compile.
145 Otherwise we wrap if and try to compile.
146
146
147 If it works, assume it should be async. Otherwise Return False.
147 If it works, assume it should be async. Otherwise Return False.
148
148
149 Not handled yet: If the block of code has a return statement as the top
149 Not handled yet: If the block of code has a return statement as the top
150 level, it will be seen as async. This is a know limitation.
150 level, it will be seen as async. This is a know limitation.
151 """
151 """
152 if sys.version_info > (3, 8):
152 if sys.version_info > (3, 8):
153 try:
153 try:
154 code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0))
154 code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0))
155 return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE
155 return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE
156 except SyntaxError:
156 except (SyntaxError, MemoryError):
157 return False
157 return False
158 try:
158 try:
159 # we can't limit ourself to ast.parse, as it __accepts__ to parse on
159 # we can't limit ourself to ast.parse, as it __accepts__ to parse on
160 # 3.7+, but just does not _compile_
160 # 3.7+, but just does not _compile_
161 code = compile(cell, "<>", "exec")
161 code = compile(cell, "<>", "exec")
162 except SyntaxError:
162 except (SyntaxError, MemoryError):
163 try:
163 try:
164 parse_tree = _async_parse_cell(cell)
164 parse_tree = _async_parse_cell(cell)
165
165
166 # Raise a SyntaxError if there are top-level return or yields
166 # Raise a SyntaxError if there are top-level return or yields
167 v = _AsyncSyntaxErrorVisitor()
167 v = _AsyncSyntaxErrorVisitor()
168 v.visit(parse_tree)
168 v.visit(parse_tree)
169
169
170 except SyntaxError:
170 except (SyntaxError, MemoryError):
171 return False
171 return False
172 return True
172 return True
173 return False
173 return False
General Comments 0
You need to be logged in to leave comments. Login now