Show More
@@ -1,77 +1,82 b'' | |||
|
1 | 1 | """ |
|
2 | 2 | Base front end class for all async frontends. |
|
3 | 3 | """ |
|
4 | 4 | __docformat__ = "restructuredtext en" |
|
5 | 5 | |
|
6 | # Tell nose to skip this module | |
|
7 | __test__ = {} | |
|
8 | ||
|
6 | 9 | #------------------------------------------------------------------------------- |
|
7 | 10 | # Copyright (C) 2008 The IPython Development Team |
|
8 | 11 | # |
|
9 | 12 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | 13 | # the file COPYING, distributed as part of this software. |
|
11 | 14 | #------------------------------------------------------------------------------- |
|
12 | 15 | |
|
13 | ||
|
14 | 16 | #------------------------------------------------------------------------------- |
|
15 | 17 | # Imports |
|
16 | 18 | #------------------------------------------------------------------------------- |
|
17 | 19 | |
|
20 | # Third-party | |
|
21 | from twisted.python.failure import Failure | |
|
22 | from zope.interface import implements, classProvides | |
|
23 | ||
|
24 | # From IPython | |
|
18 | 25 | from IPython.external import guid |
|
19 | 26 | |
|
20 | from zope.interface import Interface, Attribute, implements, classProvides | |
|
21 | from twisted.python.failure import Failure | |
|
22 | from IPython.frontend.frontendbase import ( | |
|
23 | FrontEndBase, IFrontEnd, IFrontEndFactory) | |
|
27 | from IPython.frontend.frontendbase import (FrontEndBase, IFrontEnd, | |
|
28 | IFrontEndFactory) | |
|
24 | 29 | from IPython.kernel.core.history import FrontEndHistory |
|
25 | 30 | from IPython.kernel.engineservice import IEngineCore |
|
26 | 31 | |
|
32 | #----------------------------------------------------------------------------- | |
|
33 | # Classes and functions | |
|
34 | #----------------------------------------------------------------------------- | |
|
27 | 35 | |
|
28 | 36 | class AsyncFrontEndBase(FrontEndBase): |
|
29 | 37 | """ |
|
30 | 38 | Overrides FrontEndBase to wrap execute in a deferred result. |
|
31 | 39 | All callbacks are made as callbacks on the deferred result. |
|
32 | 40 | """ |
|
33 | 41 | |
|
34 | 42 | implements(IFrontEnd) |
|
35 | 43 | classProvides(IFrontEndFactory) |
|
36 | 44 | |
|
37 | 45 | def __init__(self, engine=None, history=None): |
|
38 | 46 | assert(engine==None or IEngineCore.providedBy(engine)) |
|
39 | 47 | self.engine = IEngineCore(engine) |
|
40 | 48 | if history is None: |
|
41 | 49 | self.history = FrontEndHistory(input_cache=['']) |
|
42 | 50 | else: |
|
43 | 51 | self.history = history |
|
44 | ||
|
45 | ||
|
52 | ||
|
46 | 53 | def execute(self, block, blockID=None): |
|
47 | 54 | """Execute the block and return the deferred result. |
|
48 | 55 | |
|
49 | 56 | Parameters: |
|
50 | 57 | block : {str, AST} |
|
51 | 58 | blockID : any |
|
52 | 59 | Caller may provide an ID to identify this block. |
|
53 | 60 | result['blockID'] := blockID |
|
54 | 61 | |
|
55 | 62 | Result: |
|
56 | 63 | Deferred result of self.interpreter.execute |
|
57 | 64 | """ |
|
58 | 65 | |
|
59 | 66 | if(not self.is_complete(block)): |
|
60 | 67 | return Failure(Exception("Block is not compilable")) |
|
61 | 68 | |
|
62 | 69 | if(blockID == None): |
|
63 | 70 | blockID = guid.generate() |
|
64 | 71 | |
|
65 | 72 | d = self.engine.execute(block) |
|
66 | 73 | d.addCallback(self._add_history, block=block) |
|
67 | 74 | d.addCallbacks(self._add_block_id_for_result, |
|
68 | 75 | errback=self._add_block_id_for_failure, |
|
69 | 76 | callbackArgs=(blockID,), |
|
70 | 77 | errbackArgs=(blockID,)) |
|
71 | 78 | d.addBoth(self.update_cell_prompt, blockID=blockID) |
|
72 | 79 | d.addCallbacks(self.render_result, |
|
73 | 80 | errback=self.render_error) |
|
74 | 81 | |
|
75 | 82 | return d |
|
76 | ||
|
77 |
@@ -1,109 +1,112 b'' | |||
|
1 | 1 | # encoding: utf-8 |
|
2 | 2 | |
|
3 | 3 | """This file contains unittests for the asyncfrontendbase module.""" |
|
4 | 4 | |
|
5 | 5 | __docformat__ = "restructuredtext en" |
|
6 | ||
|
7 | # Tell nose to skip this module | |
|
8 | __test__ = {} | |
|
6 | 9 | |
|
7 | 10 | #--------------------------------------------------------------------------- |
|
8 | 11 | # Copyright (C) 2008 The IPython Development Team |
|
9 | 12 | # |
|
10 | 13 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | 14 | # the file COPYING, distributed as part of this software. |
|
12 | 15 | #--------------------------------------------------------------------------- |
|
13 | ||
|
16 | ||
|
14 | 17 | #--------------------------------------------------------------------------- |
|
15 | 18 | # Imports |
|
16 | 19 | #--------------------------------------------------------------------------- |
|
17 | 20 | |
|
18 | # Tell nose to skip this module | |
|
19 | __test__ = {} | |
|
20 | ||
|
21 | 21 | from twisted.trial import unittest |
|
22 | ||
|
22 | 23 | from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase |
|
23 | 24 | from IPython.frontend import frontendbase |
|
24 | 25 | from IPython.kernel.engineservice import EngineService |
|
25 | 26 | from IPython.testing.parametric import Parametric, parametric |
|
26 | 27 | |
|
28 | #----------------------------------------------------------------------------- | |
|
29 | # Classes and functions | |
|
30 | #----------------------------------------------------------------------------- | |
|
27 | 31 | |
|
28 | 32 | class FrontEndCallbackChecker(AsyncFrontEndBase): |
|
29 | 33 | """FrontEndBase subclass for checking callbacks""" |
|
30 | 34 | def __init__(self, engine=None, history=None): |
|
31 | 35 | super(FrontEndCallbackChecker, self).__init__(engine=engine, |
|
32 | 36 | history=history) |
|
33 | 37 | self.updateCalled = False |
|
34 | 38 | self.renderResultCalled = False |
|
35 | 39 | self.renderErrorCalled = False |
|
36 | 40 | |
|
37 | 41 | def update_cell_prompt(self, result, blockID=None): |
|
38 | 42 | self.updateCalled = True |
|
39 | 43 | return result |
|
40 | 44 | |
|
41 | 45 | def render_result(self, result): |
|
42 | 46 | self.renderResultCalled = True |
|
43 | 47 | return result |
|
44 | 48 | |
|
45 | 49 | def render_error(self, failure): |
|
46 | 50 | self.renderErrorCalled = True |
|
47 | 51 | return failure |
|
48 | 52 | |
|
49 | 53 | |
|
50 | 54 | class TestAsyncFrontendBase(unittest.TestCase): |
|
51 | 55 | def setUp(self): |
|
52 | 56 | """Setup the EngineService and FrontEndBase""" |
|
53 | 57 | |
|
54 | 58 | self.fb = FrontEndCallbackChecker(engine=EngineService()) |
|
55 | 59 | |
|
56 | 60 | def test_implements_IFrontEnd(self): |
|
57 | 61 | self.assert_(frontendbase.IFrontEnd.implementedBy( |
|
58 | 62 | AsyncFrontEndBase)) |
|
59 | 63 | |
|
60 | 64 | def test_is_complete_returns_False_for_incomplete_block(self): |
|
61 | 65 | block = """def test(a):""" |
|
62 | 66 | self.assert_(self.fb.is_complete(block) == False) |
|
63 | 67 | |
|
64 | 68 | def test_is_complete_returns_True_for_complete_block(self): |
|
65 | 69 | block = """def test(a): pass""" |
|
66 | 70 | self.assert_(self.fb.is_complete(block)) |
|
67 | 71 | block = """a=3""" |
|
68 | 72 | self.assert_(self.fb.is_complete(block)) |
|
69 | 73 | |
|
70 | 74 | def test_blockID_added_to_result(self): |
|
71 | 75 | block = """3+3""" |
|
72 | 76 | d = self.fb.execute(block, blockID='TEST_ID') |
|
73 | 77 | d.addCallback(lambda r: self.assert_(r['blockID']=='TEST_ID')) |
|
74 | 78 | return d |
|
75 | 79 | |
|
76 | 80 | def test_blockID_added_to_failure(self): |
|
77 | 81 | block = "raise Exception()" |
|
78 | 82 | d = self.fb.execute(block,blockID='TEST_ID') |
|
79 | 83 | d.addErrback(lambda f: self.assert_(f.blockID=='TEST_ID')) |
|
80 | 84 | return d |
|
81 | 85 | |
|
82 | 86 | def test_callbacks_added_to_execute(self): |
|
83 | 87 | d = self.fb.execute("10+10") |
|
84 | 88 | d.addCallback(lambda r: self.assert_(self.fb.updateCalled and self.fb.renderResultCalled)) |
|
85 | 89 | return d |
|
86 | 90 | |
|
87 | 91 | def test_error_callback_added_to_execute(self): |
|
88 | 92 | """Test that render_error called on execution error.""" |
|
89 | 93 | |
|
90 | 94 | d = self.fb.execute("raise Exception()") |
|
91 | 95 | d.addErrback(lambda f: self.assert_(self.fb.renderErrorCalled)) |
|
92 | 96 | return d |
|
93 | 97 | |
|
94 | 98 | def test_history_returns_expected_block(self): |
|
95 | 99 | """Make sure history browsing doesn't fail.""" |
|
96 | 100 | |
|
97 | 101 | blocks = ["a=1","a=2","a=3"] |
|
98 | 102 | d = self.fb.execute(blocks[0]) |
|
99 | 103 | d.addCallback(lambda _: self.fb.execute(blocks[1])) |
|
100 | 104 | d.addCallback(lambda _: self.fb.execute(blocks[2])) |
|
101 | 105 | d.addCallback(lambda _: self.assert_(self.fb.get_history_previous("")==blocks[-2])) |
|
102 | 106 | d.addCallback(lambda _: self.assert_(self.fb.get_history_previous("")==blocks[-3])) |
|
103 | 107 | d.addCallback(lambda _: self.assert_(self.fb.get_history_next()==blocks[-2])) |
|
104 | 108 | return d |
|
105 | 109 | |
|
106 | 110 | def test_history_returns_none_at_startup(self): |
|
107 | 111 | self.assert_(self.fb.get_history_previous("")==None) |
|
108 | 112 | self.assert_(self.fb.get_history_next()==None) |
|
109 |
@@ -1,906 +1,902 b'' | |||
|
1 | 1 | # encoding: utf-8 |
|
2 | 2 | # -*- test-case-name: IPython.kernel.tests.test_engineservice -*- |
|
3 | 3 | |
|
4 | 4 | """A Twisted Service Representation of the IPython core. |
|
5 | 5 | |
|
6 | 6 | The IPython Core exposed to the network is called the Engine. Its |
|
7 | 7 | representation in Twisted in the EngineService. Interfaces and adapters |
|
8 | 8 | are used to abstract out the details of the actual network protocol used. |
|
9 | 9 | The EngineService is an Engine that knows nothing about the actual protocol |
|
10 | 10 | used. |
|
11 | 11 | |
|
12 | 12 | The EngineService is exposed with various network protocols in modules like: |
|
13 | 13 | |
|
14 | 14 | enginepb.py |
|
15 | 15 | enginevanilla.py |
|
16 | 16 | |
|
17 | 17 | As of 12/12/06 the classes in this module have been simplified greatly. It was |
|
18 | 18 | felt that we had over-engineered things. To improve the maintainability of the |
|
19 | 19 | code we have taken out the ICompleteEngine interface and the completeEngine |
|
20 | 20 | method that automatically added methods to engines. |
|
21 | 21 | |
|
22 | 22 | """ |
|
23 | 23 | |
|
24 | 24 | __docformat__ = "restructuredtext en" |
|
25 | 25 | |
|
26 | # Tell nose to skip this module | |
|
27 | __test__ = {} | |
|
28 | ||
|
26 | 29 | #------------------------------------------------------------------------------- |
|
27 | 30 | # Copyright (C) 2008 The IPython Development Team |
|
28 | 31 | # |
|
29 | 32 | # Distributed under the terms of the BSD License. The full license is in |
|
30 | 33 | # the file COPYING, distributed as part of this software. |
|
31 | 34 | #------------------------------------------------------------------------------- |
|
32 | 35 | |
|
33 | 36 | #------------------------------------------------------------------------------- |
|
34 | 37 | # Imports |
|
35 | 38 | #------------------------------------------------------------------------------- |
|
36 | 39 | |
|
37 | # Tell nose to skip the testing of this module | |
|
38 | __test__ = {} | |
|
39 | ||
|
40 | import os, sys, copy | |
|
40 | import copy | |
|
41 | import sys | |
|
41 | 42 | import cPickle as pickle |
|
42 | from new import instancemethod | |
|
43 | 43 | |
|
44 | 44 | from twisted.application import service |
|
45 | 45 | from twisted.internet import defer, reactor |
|
46 | 46 | from twisted.python import log, failure, components |
|
47 | 47 | import zope.interface as zi |
|
48 | 48 | |
|
49 | 49 | from IPython.kernel.core.interpreter import Interpreter |
|
50 |
from IPython.kernel import newserialized, error |
|
|
51 | from IPython.kernel.util import printer | |
|
52 | from IPython.kernel.twistedutil import gatherBoth, DeferredList | |
|
53 | from IPython.kernel import codeutil | |
|
54 | ||
|
50 | from IPython.kernel import newserialized, error | |
|
55 | 51 | |
|
56 | 52 | #------------------------------------------------------------------------------- |
|
57 | 53 | # Interface specification for the Engine |
|
58 | 54 | #------------------------------------------------------------------------------- |
|
59 | 55 | |
|
60 | 56 | class IEngineCore(zi.Interface): |
|
61 | 57 | """The minimal required interface for the IPython Engine. |
|
62 | 58 | |
|
63 | 59 | This interface provides a formal specification of the IPython core. |
|
64 | 60 | All these methods should return deferreds regardless of what side of a |
|
65 | 61 | network connection they are on. |
|
66 | 62 | |
|
67 | 63 | In general, this class simply wraps a shell class and wraps its return |
|
68 | 64 | values as Deferred objects. If the underlying shell class method raises |
|
69 | 65 | an exception, this class should convert it to a twisted.failure.Failure |
|
70 | 66 | that will be propagated along the Deferred's errback chain. |
|
71 | 67 | |
|
72 | 68 | In addition, Failures are aggressive. By this, we mean that if a method |
|
73 | 69 | is performing multiple actions (like pulling multiple object) if any |
|
74 | 70 | single one fails, the entire method will fail with that Failure. It is |
|
75 | 71 | all or nothing. |
|
76 | 72 | """ |
|
77 | 73 | |
|
78 | 74 | id = zi.interface.Attribute("the id of the Engine object") |
|
79 | 75 | properties = zi.interface.Attribute("A dict of properties of the Engine") |
|
80 | 76 | |
|
81 | 77 | def execute(lines): |
|
82 | 78 | """Execute lines of Python code. |
|
83 | 79 | |
|
84 | 80 | Returns a dictionary with keys (id, number, stdin, stdout, stderr) |
|
85 | 81 | upon success. |
|
86 | 82 | |
|
87 | 83 | Returns a failure object if the execution of lines raises an exception. |
|
88 | 84 | """ |
|
89 | 85 | |
|
90 | 86 | def push(namespace): |
|
91 | 87 | """Push dict namespace into the user's namespace. |
|
92 | 88 | |
|
93 | 89 | Returns a deferred to None or a failure. |
|
94 | 90 | """ |
|
95 | 91 | |
|
96 | 92 | def pull(keys): |
|
97 | 93 | """Pulls values out of the user's namespace by keys. |
|
98 | 94 | |
|
99 | 95 | Returns a deferred to a tuple objects or a single object. |
|
100 | 96 | |
|
101 | 97 | Raises NameError if any one of objects doess not exist. |
|
102 | 98 | """ |
|
103 | 99 | |
|
104 | 100 | def push_function(namespace): |
|
105 | 101 | """Push a dict of key, function pairs into the user's namespace. |
|
106 | 102 | |
|
107 | 103 | Returns a deferred to None or a failure.""" |
|
108 | 104 | |
|
109 | 105 | def pull_function(keys): |
|
110 | 106 | """Pulls functions out of the user's namespace by keys. |
|
111 | 107 | |
|
112 | 108 | Returns a deferred to a tuple of functions or a single function. |
|
113 | 109 | |
|
114 | 110 | Raises NameError if any one of the functions does not exist. |
|
115 | 111 | """ |
|
116 | 112 | |
|
117 | 113 | def get_result(i=None): |
|
118 | 114 | """Get the stdin/stdout/stderr of command i. |
|
119 | 115 | |
|
120 | 116 | Returns a deferred to a dict with keys |
|
121 | 117 | (id, number, stdin, stdout, stderr). |
|
122 | 118 | |
|
123 | 119 | Raises IndexError if command i does not exist. |
|
124 | 120 | Raises TypeError if i in not an int. |
|
125 | 121 | """ |
|
126 | 122 | |
|
127 | 123 | def reset(): |
|
128 | 124 | """Reset the shell. |
|
129 | 125 | |
|
130 | 126 | This clears the users namespace. Won't cause modules to be |
|
131 | 127 | reloaded. Should also re-initialize certain variables like id. |
|
132 | 128 | """ |
|
133 | 129 | |
|
134 | 130 | def kill(): |
|
135 | 131 | """Kill the engine by stopping the reactor.""" |
|
136 | 132 | |
|
137 | 133 | def keys(): |
|
138 | 134 | """Return the top level variables in the users namspace. |
|
139 | 135 | |
|
140 | 136 | Returns a deferred to a dict.""" |
|
141 | 137 | |
|
142 | 138 | |
|
143 | 139 | class IEngineSerialized(zi.Interface): |
|
144 | 140 | """Push/Pull methods that take Serialized objects. |
|
145 | 141 | |
|
146 | 142 | All methods should return deferreds. |
|
147 | 143 | """ |
|
148 | 144 | |
|
149 | 145 | def push_serialized(namespace): |
|
150 | 146 | """Push a dict of keys and Serialized objects into the user's namespace.""" |
|
151 | 147 | |
|
152 | 148 | def pull_serialized(keys): |
|
153 | 149 | """Pull objects by key from the user's namespace as Serialized. |
|
154 | 150 | |
|
155 | 151 | Returns a list of or one Serialized. |
|
156 | 152 | |
|
157 | 153 | Raises NameError is any one of the objects does not exist. |
|
158 | 154 | """ |
|
159 | 155 | |
|
160 | 156 | |
|
161 | 157 | class IEngineProperties(zi.Interface): |
|
162 | 158 | """Methods for access to the properties object of an Engine""" |
|
163 | 159 | |
|
164 | 160 | properties = zi.Attribute("A StrictDict object, containing the properties") |
|
165 | 161 | |
|
166 | 162 | def set_properties(properties): |
|
167 | 163 | """set properties by key and value""" |
|
168 | 164 | |
|
169 | 165 | def get_properties(keys=None): |
|
170 | 166 | """get a list of properties by `keys`, if no keys specified, get all""" |
|
171 | 167 | |
|
172 | 168 | def del_properties(keys): |
|
173 | 169 | """delete properties by `keys`""" |
|
174 | 170 | |
|
175 | 171 | def has_properties(keys): |
|
176 | 172 | """get a list of bool values for whether `properties` has `keys`""" |
|
177 | 173 | |
|
178 | 174 | def clear_properties(): |
|
179 | 175 | """clear the properties dict""" |
|
180 | 176 | |
|
181 | 177 | class IEngineBase(IEngineCore, IEngineSerialized, IEngineProperties): |
|
182 | 178 | """The basic engine interface that EngineService will implement. |
|
183 | 179 | |
|
184 | 180 | This exists so it is easy to specify adapters that adapt to and from the |
|
185 | 181 | API that the basic EngineService implements. |
|
186 | 182 | """ |
|
187 | 183 | pass |
|
188 | 184 | |
|
189 | 185 | class IEngineQueued(IEngineBase): |
|
190 | 186 | """Interface for adding a queue to an IEngineBase. |
|
191 | 187 | |
|
192 | 188 | This interface extends the IEngineBase interface to add methods for managing |
|
193 | 189 | the engine's queue. The implicit details of this interface are that the |
|
194 | 190 | execution of all methods declared in IEngineBase should appropriately be |
|
195 | 191 | put through a queue before execution. |
|
196 | 192 | |
|
197 | 193 | All methods should return deferreds. |
|
198 | 194 | """ |
|
199 | 195 | |
|
200 | 196 | def clear_queue(): |
|
201 | 197 | """Clear the queue.""" |
|
202 | 198 | |
|
203 | 199 | def queue_status(): |
|
204 | 200 | """Get the queued and pending commands in the queue.""" |
|
205 | 201 | |
|
206 | 202 | def register_failure_observer(obs): |
|
207 | 203 | """Register an observer of pending Failures. |
|
208 | 204 | |
|
209 | 205 | The observer must implement IFailureObserver. |
|
210 | 206 | """ |
|
211 | 207 | |
|
212 | 208 | def unregister_failure_observer(obs): |
|
213 | 209 | """Unregister an observer of pending Failures.""" |
|
214 | 210 | |
|
215 | 211 | |
|
216 | 212 | class IEngineThreaded(zi.Interface): |
|
217 | 213 | """A place holder for threaded commands. |
|
218 | 214 | |
|
219 | 215 | All methods should return deferreds. |
|
220 | 216 | """ |
|
221 | 217 | pass |
|
222 | 218 | |
|
223 | 219 | |
|
224 | 220 | #------------------------------------------------------------------------------- |
|
225 | 221 | # Functions and classes to implement the EngineService |
|
226 | 222 | #------------------------------------------------------------------------------- |
|
227 | 223 | |
|
228 | 224 | |
|
229 | 225 | class StrictDict(dict): |
|
230 | 226 | """This is a strict copying dictionary for use as the interface to the |
|
231 | 227 | properties of an Engine. |
|
232 | 228 | |
|
233 | 229 | :IMPORTANT: |
|
234 | 230 | This object copies the values you set to it, and returns copies to you |
|
235 | 231 | when you request them. The only way to change properties os explicitly |
|
236 | 232 | through the setitem and getitem of the dictionary interface. |
|
237 | 233 | |
|
238 | 234 | Example: |
|
239 | 235 | >>> e = get_engine(id) |
|
240 | 236 | >>> L = [1,2,3] |
|
241 | 237 | >>> e.properties['L'] = L |
|
242 | 238 | >>> L == e.properties['L'] |
|
243 | 239 | True |
|
244 | 240 | >>> L.append(99) |
|
245 | 241 | >>> L == e.properties['L'] |
|
246 | 242 | False |
|
247 | 243 | |
|
248 | 244 | Note that getitem copies, so calls to methods of objects do not affect |
|
249 | 245 | the properties, as seen here: |
|
250 | 246 | |
|
251 | 247 | >>> e.properties[1] = range(2) |
|
252 | 248 | >>> print e.properties[1] |
|
253 | 249 | [0, 1] |
|
254 | 250 | >>> e.properties[1].append(2) |
|
255 | 251 | >>> print e.properties[1] |
|
256 | 252 | [0, 1] |
|
257 | 253 | """ |
|
258 | 254 | def __init__(self, *args, **kwargs): |
|
259 | 255 | dict.__init__(self, *args, **kwargs) |
|
260 | 256 | self.modified = True |
|
261 | 257 | |
|
262 | 258 | def __getitem__(self, key): |
|
263 | 259 | return copy.deepcopy(dict.__getitem__(self, key)) |
|
264 | 260 | |
|
265 | 261 | def __setitem__(self, key, value): |
|
266 | 262 | # check if this entry is valid for transport around the network |
|
267 | 263 | # and copying |
|
268 | 264 | try: |
|
269 | 265 | pickle.dumps(key, 2) |
|
270 | 266 | pickle.dumps(value, 2) |
|
271 | 267 | newvalue = copy.deepcopy(value) |
|
272 | 268 | except Exception, e: |
|
273 | 269 | raise error.InvalidProperty("can't be a value: %r" % value) |
|
274 | 270 | dict.__setitem__(self, key, newvalue) |
|
275 | 271 | self.modified = True |
|
276 | 272 | |
|
277 | 273 | def __delitem__(self, key): |
|
278 | 274 | dict.__delitem__(self, key) |
|
279 | 275 | self.modified = True |
|
280 | 276 | |
|
281 | 277 | def update(self, dikt): |
|
282 | 278 | for k,v in dikt.iteritems(): |
|
283 | 279 | self[k] = v |
|
284 | 280 | |
|
285 | 281 | def pop(self, key): |
|
286 | 282 | self.modified = True |
|
287 | 283 | return dict.pop(self, key) |
|
288 | 284 | |
|
289 | 285 | def popitem(self): |
|
290 | 286 | self.modified = True |
|
291 | 287 | return dict.popitem(self) |
|
292 | 288 | |
|
293 | 289 | def clear(self): |
|
294 | 290 | self.modified = True |
|
295 | 291 | dict.clear(self) |
|
296 | 292 | |
|
297 | 293 | def subDict(self, *keys): |
|
298 | 294 | d = {} |
|
299 | 295 | for key in keys: |
|
300 | 296 | d[key] = self[key] |
|
301 | 297 | return d |
|
302 | 298 | |
|
303 | 299 | |
|
304 | 300 | |
|
305 | 301 | class EngineAPI(object): |
|
306 | 302 | """This is the object through which the user can edit the `properties` |
|
307 | 303 | attribute of an Engine. |
|
308 | 304 | The Engine Properties object copies all object in and out of itself. |
|
309 | 305 | See the EngineProperties object for details. |
|
310 | 306 | """ |
|
311 | 307 | _fix=False |
|
312 | 308 | def __init__(self, id): |
|
313 | 309 | self.id = id |
|
314 | 310 | self.properties = StrictDict() |
|
315 | 311 | self._fix=True |
|
316 | 312 | |
|
317 | 313 | def __setattr__(self, k,v): |
|
318 | 314 | if self._fix: |
|
319 | 315 | raise error.KernelError("I am protected!") |
|
320 | 316 | else: |
|
321 | 317 | object.__setattr__(self, k, v) |
|
322 | 318 | |
|
323 | 319 | def __delattr__(self, key): |
|
324 | 320 | raise error.KernelError("I am protected!") |
|
325 | 321 | |
|
326 | 322 | |
|
327 | 323 | _apiDict = {} |
|
328 | 324 | |
|
329 | 325 | def get_engine(id): |
|
330 | 326 | """Get the Engine API object, whcih currently just provides the properties |
|
331 | 327 | object, by ID""" |
|
332 | 328 | global _apiDict |
|
333 | 329 | if not _apiDict.get(id): |
|
334 | 330 | _apiDict[id] = EngineAPI(id) |
|
335 | 331 | return _apiDict[id] |
|
336 | 332 | |
|
337 | 333 | def drop_engine(id): |
|
338 | 334 | """remove an engine""" |
|
339 | 335 | global _apiDict |
|
340 | 336 | if _apiDict.has_key(id): |
|
341 | 337 | del _apiDict[id] |
|
342 | 338 | |
|
343 | 339 | class EngineService(object, service.Service): |
|
344 | 340 | """Adapt a IPython shell into a IEngine implementing Twisted Service.""" |
|
345 | 341 | |
|
346 | 342 | zi.implements(IEngineBase) |
|
347 | 343 | name = 'EngineService' |
|
348 | 344 | |
|
349 | 345 | def __init__(self, shellClass=Interpreter, mpi=None): |
|
350 | 346 | """Create an EngineService. |
|
351 | 347 | |
|
352 | 348 | shellClass: something that implements IInterpreter or core1 |
|
353 | 349 | mpi: an mpi module that has rank and size attributes |
|
354 | 350 | """ |
|
355 | 351 | self.shellClass = shellClass |
|
356 | 352 | self.shell = self.shellClass() |
|
357 | 353 | self.mpi = mpi |
|
358 | 354 | self.id = None |
|
359 | 355 | self.properties = get_engine(self.id).properties |
|
360 | 356 | if self.mpi is not None: |
|
361 | 357 | log.msg("MPI started with rank = %i and size = %i" % |
|
362 | 358 | (self.mpi.rank, self.mpi.size)) |
|
363 | 359 | self.id = self.mpi.rank |
|
364 | 360 | self._seedNamespace() |
|
365 | 361 | |
|
366 | 362 | # Make id a property so that the shell can get the updated id |
|
367 | 363 | |
|
368 | 364 | def _setID(self, id): |
|
369 | 365 | self._id = id |
|
370 | 366 | self.properties = get_engine(id).properties |
|
371 | 367 | self.shell.push({'id': id}) |
|
372 | 368 | |
|
373 | 369 | def _getID(self): |
|
374 | 370 | return self._id |
|
375 | 371 | |
|
376 | 372 | id = property(_getID, _setID) |
|
377 | 373 | |
|
378 | 374 | def _seedNamespace(self): |
|
379 | 375 | self.shell.push({'mpi': self.mpi, 'id' : self.id}) |
|
380 | 376 | |
|
381 | 377 | def executeAndRaise(self, msg, callable, *args, **kwargs): |
|
382 | 378 | """Call a method of self.shell and wrap any exception.""" |
|
383 | 379 | d = defer.Deferred() |
|
384 | 380 | try: |
|
385 | 381 | result = callable(*args, **kwargs) |
|
386 | 382 | except: |
|
387 | 383 | # This gives the following: |
|
388 | 384 | # et=exception class |
|
389 | 385 | # ev=exception class instance |
|
390 | 386 | # tb=traceback object |
|
391 | 387 | et,ev,tb = sys.exc_info() |
|
392 | 388 | # This call adds attributes to the exception value |
|
393 | 389 | et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg) |
|
394 | 390 | # Add another attribute |
|
395 | 391 | ev._ipython_engine_info = msg |
|
396 | 392 | f = failure.Failure(ev,et,None) |
|
397 | 393 | d.errback(f) |
|
398 | 394 | else: |
|
399 | 395 | d.callback(result) |
|
400 | 396 | |
|
401 | 397 | return d |
|
402 | 398 | |
|
403 | 399 | |
|
404 | 400 | # The IEngine methods. See the interface for documentation. |
|
405 | 401 | |
|
406 | 402 | def execute(self, lines): |
|
407 | 403 | msg = {'engineid':self.id, |
|
408 | 404 | 'method':'execute', |
|
409 | 405 | 'args':[lines]} |
|
410 | 406 | d = self.executeAndRaise(msg, self.shell.execute, lines) |
|
411 | 407 | d.addCallback(self.addIDToResult) |
|
412 | 408 | return d |
|
413 | 409 | |
|
414 | 410 | def addIDToResult(self, result): |
|
415 | 411 | result['id'] = self.id |
|
416 | 412 | return result |
|
417 | 413 | |
|
418 | 414 | def push(self, namespace): |
|
419 | 415 | msg = {'engineid':self.id, |
|
420 | 416 | 'method':'push', |
|
421 | 417 | 'args':[repr(namespace.keys())]} |
|
422 | 418 | d = self.executeAndRaise(msg, self.shell.push, namespace) |
|
423 | 419 | return d |
|
424 | 420 | |
|
425 | 421 | def pull(self, keys): |
|
426 | 422 | msg = {'engineid':self.id, |
|
427 | 423 | 'method':'pull', |
|
428 | 424 | 'args':[repr(keys)]} |
|
429 | 425 | d = self.executeAndRaise(msg, self.shell.pull, keys) |
|
430 | 426 | return d |
|
431 | 427 | |
|
432 | 428 | def push_function(self, namespace): |
|
433 | 429 | msg = {'engineid':self.id, |
|
434 | 430 | 'method':'push_function', |
|
435 | 431 | 'args':[repr(namespace.keys())]} |
|
436 | 432 | d = self.executeAndRaise(msg, self.shell.push_function, namespace) |
|
437 | 433 | return d |
|
438 | 434 | |
|
439 | 435 | def pull_function(self, keys): |
|
440 | 436 | msg = {'engineid':self.id, |
|
441 | 437 | 'method':'pull_function', |
|
442 | 438 | 'args':[repr(keys)]} |
|
443 | 439 | d = self.executeAndRaise(msg, self.shell.pull_function, keys) |
|
444 | 440 | return d |
|
445 | 441 | |
|
446 | 442 | def get_result(self, i=None): |
|
447 | 443 | msg = {'engineid':self.id, |
|
448 | 444 | 'method':'get_result', |
|
449 | 445 | 'args':[repr(i)]} |
|
450 | 446 | d = self.executeAndRaise(msg, self.shell.getCommand, i) |
|
451 | 447 | d.addCallback(self.addIDToResult) |
|
452 | 448 | return d |
|
453 | 449 | |
|
454 | 450 | def reset(self): |
|
455 | 451 | msg = {'engineid':self.id, |
|
456 | 452 | 'method':'reset', |
|
457 | 453 | 'args':[]} |
|
458 | 454 | del self.shell |
|
459 | 455 | self.shell = self.shellClass() |
|
460 | 456 | self.properties.clear() |
|
461 | 457 | d = self.executeAndRaise(msg, self._seedNamespace) |
|
462 | 458 | return d |
|
463 | 459 | |
|
464 | 460 | def kill(self): |
|
465 | 461 | drop_engine(self.id) |
|
466 | 462 | try: |
|
467 | 463 | reactor.stop() |
|
468 | 464 | except RuntimeError: |
|
469 | 465 | log.msg('The reactor was not running apparently.') |
|
470 | 466 | return defer.fail() |
|
471 | 467 | else: |
|
472 | 468 | return defer.succeed(None) |
|
473 | 469 | |
|
474 | 470 | def keys(self): |
|
475 | 471 | """Return a list of variables names in the users top level namespace. |
|
476 | 472 | |
|
477 | 473 | This used to return a dict of all the keys/repr(values) in the |
|
478 | 474 | user's namespace. This was too much info for the ControllerService |
|
479 | 475 | to handle so it is now just a list of keys. |
|
480 | 476 | """ |
|
481 | 477 | |
|
482 | 478 | remotes = [] |
|
483 | 479 | for k in self.shell.user_ns.iterkeys(): |
|
484 | 480 | if k not in ['__name__', '_ih', '_oh', '__builtins__', |
|
485 | 481 | 'In', 'Out', '_', '__', '___', '__IP', 'input', 'raw_input']: |
|
486 | 482 | remotes.append(k) |
|
487 | 483 | return defer.succeed(remotes) |
|
488 | 484 | |
|
489 | 485 | def set_properties(self, properties): |
|
490 | 486 | msg = {'engineid':self.id, |
|
491 | 487 | 'method':'set_properties', |
|
492 | 488 | 'args':[repr(properties.keys())]} |
|
493 | 489 | return self.executeAndRaise(msg, self.properties.update, properties) |
|
494 | 490 | |
|
495 | 491 | def get_properties(self, keys=None): |
|
496 | 492 | msg = {'engineid':self.id, |
|
497 | 493 | 'method':'get_properties', |
|
498 | 494 | 'args':[repr(keys)]} |
|
499 | 495 | if keys is None: |
|
500 | 496 | keys = self.properties.keys() |
|
501 | 497 | return self.executeAndRaise(msg, self.properties.subDict, *keys) |
|
502 | 498 | |
|
503 | 499 | def _doDel(self, keys): |
|
504 | 500 | for key in keys: |
|
505 | 501 | del self.properties[key] |
|
506 | 502 | |
|
507 | 503 | def del_properties(self, keys): |
|
508 | 504 | msg = {'engineid':self.id, |
|
509 | 505 | 'method':'del_properties', |
|
510 | 506 | 'args':[repr(keys)]} |
|
511 | 507 | return self.executeAndRaise(msg, self._doDel, keys) |
|
512 | 508 | |
|
513 | 509 | def _doHas(self, keys): |
|
514 | 510 | return [self.properties.has_key(key) for key in keys] |
|
515 | 511 | |
|
516 | 512 | def has_properties(self, keys): |
|
517 | 513 | msg = {'engineid':self.id, |
|
518 | 514 | 'method':'has_properties', |
|
519 | 515 | 'args':[repr(keys)]} |
|
520 | 516 | return self.executeAndRaise(msg, self._doHas, keys) |
|
521 | 517 | |
|
522 | 518 | def clear_properties(self): |
|
523 | 519 | msg = {'engineid':self.id, |
|
524 | 520 | 'method':'clear_properties', |
|
525 | 521 | 'args':[]} |
|
526 | 522 | return self.executeAndRaise(msg, self.properties.clear) |
|
527 | 523 | |
|
528 | 524 | def push_serialized(self, sNamespace): |
|
529 | 525 | msg = {'engineid':self.id, |
|
530 | 526 | 'method':'push_serialized', |
|
531 | 527 | 'args':[repr(sNamespace.keys())]} |
|
532 | 528 | ns = {} |
|
533 | 529 | for k,v in sNamespace.iteritems(): |
|
534 | 530 | try: |
|
535 | 531 | unserialized = newserialized.IUnSerialized(v) |
|
536 | 532 | ns[k] = unserialized.getObject() |
|
537 | 533 | except: |
|
538 | 534 | return defer.fail() |
|
539 | 535 | return self.executeAndRaise(msg, self.shell.push, ns) |
|
540 | 536 | |
|
541 | 537 | def pull_serialized(self, keys): |
|
542 | 538 | msg = {'engineid':self.id, |
|
543 | 539 | 'method':'pull_serialized', |
|
544 | 540 | 'args':[repr(keys)]} |
|
545 | 541 | if isinstance(keys, str): |
|
546 | 542 | keys = [keys] |
|
547 | 543 | if len(keys)==1: |
|
548 | 544 | d = self.executeAndRaise(msg, self.shell.pull, keys) |
|
549 | 545 | d.addCallback(newserialized.serialize) |
|
550 | 546 | return d |
|
551 | 547 | elif len(keys)>1: |
|
552 | 548 | d = self.executeAndRaise(msg, self.shell.pull, keys) |
|
553 | 549 | @d.addCallback |
|
554 | 550 | def packThemUp(values): |
|
555 | 551 | serials = [] |
|
556 | 552 | for v in values: |
|
557 | 553 | try: |
|
558 | 554 | serials.append(newserialized.serialize(v)) |
|
559 | 555 | except: |
|
560 | 556 | return defer.fail(failure.Failure()) |
|
561 | 557 | return serials |
|
562 | 558 | return packThemUp |
|
563 | 559 | |
|
564 | 560 | |
|
565 | 561 | def queue(methodToQueue): |
|
566 | 562 | def queuedMethod(this, *args, **kwargs): |
|
567 | 563 | name = methodToQueue.__name__ |
|
568 | 564 | return this.submitCommand(Command(name, *args, **kwargs)) |
|
569 | 565 | return queuedMethod |
|
570 | 566 | |
|
571 | 567 | class QueuedEngine(object): |
|
572 | 568 | """Adapt an IEngineBase to an IEngineQueued by wrapping it. |
|
573 | 569 | |
|
574 | 570 | The resulting object will implement IEngineQueued which extends |
|
575 | 571 | IEngineCore which extends (IEngineBase, IEngineSerialized). |
|
576 | 572 | |
|
577 | 573 | This seems like the best way of handling it, but I am not sure. The |
|
578 | 574 | other option is to have the various base interfaces be used like |
|
579 | 575 | mix-in intefaces. The problem I have with this is adpatation is |
|
580 | 576 | more difficult and complicated because there can be can multiple |
|
581 | 577 | original and final Interfaces. |
|
582 | 578 | """ |
|
583 | 579 | |
|
584 | 580 | zi.implements(IEngineQueued) |
|
585 | 581 | |
|
586 | 582 | def __init__(self, engine): |
|
587 | 583 | """Create a QueuedEngine object from an engine |
|
588 | 584 | |
|
589 | 585 | engine: An implementor of IEngineCore and IEngineSerialized |
|
590 | 586 | keepUpToDate: whether to update the remote status when the |
|
591 | 587 | queue is empty. Defaults to False. |
|
592 | 588 | """ |
|
593 | 589 | |
|
594 | 590 | # This is the right way to do these tests rather than |
|
595 | 591 | # IEngineCore in list(zi.providedBy(engine)) which will only |
|
596 | 592 | # picks of the interfaces that are directly declared by engine. |
|
597 | 593 | assert IEngineBase.providedBy(engine), \ |
|
598 | 594 | "engine passed to QueuedEngine doesn't provide IEngineBase" |
|
599 | 595 | |
|
600 | 596 | self.engine = engine |
|
601 | 597 | self.id = engine.id |
|
602 | 598 | self.queued = [] |
|
603 | 599 | self.history = {} |
|
604 | 600 | self.engineStatus = {} |
|
605 | 601 | self.currentCommand = None |
|
606 | 602 | self.failureObservers = [] |
|
607 | 603 | |
|
608 | 604 | def _get_properties(self): |
|
609 | 605 | return self.engine.properties |
|
610 | 606 | |
|
611 | 607 | properties = property(_get_properties, lambda self, _: None) |
|
612 | 608 | # Queue management methods. You should not call these directly |
|
613 | 609 | |
|
614 | 610 | def submitCommand(self, cmd): |
|
615 | 611 | """Submit command to queue.""" |
|
616 | 612 | |
|
617 | 613 | d = defer.Deferred() |
|
618 | 614 | cmd.setDeferred(d) |
|
619 | 615 | if self.currentCommand is not None: |
|
620 | 616 | if self.currentCommand.finished: |
|
621 | 617 | # log.msg("Running command immediately: %r" % cmd) |
|
622 | 618 | self.currentCommand = cmd |
|
623 | 619 | self.runCurrentCommand() |
|
624 | 620 | else: # command is still running |
|
625 | 621 | # log.msg("Command is running: %r" % self.currentCommand) |
|
626 | 622 | # log.msg("Queueing: %r" % cmd) |
|
627 | 623 | self.queued.append(cmd) |
|
628 | 624 | else: |
|
629 | 625 | # log.msg("No current commands, running: %r" % cmd) |
|
630 | 626 | self.currentCommand = cmd |
|
631 | 627 | self.runCurrentCommand() |
|
632 | 628 | return d |
|
633 | 629 | |
|
634 | 630 | def runCurrentCommand(self): |
|
635 | 631 | """Run current command.""" |
|
636 | 632 | |
|
637 | 633 | cmd = self.currentCommand |
|
638 | 634 | f = getattr(self.engine, cmd.remoteMethod, None) |
|
639 | 635 | if f: |
|
640 | 636 | d = f(*cmd.args, **cmd.kwargs) |
|
641 | 637 | if cmd.remoteMethod is 'execute': |
|
642 | 638 | d.addCallback(self.saveResult) |
|
643 | 639 | d.addCallback(self.finishCommand) |
|
644 | 640 | d.addErrback(self.abortCommand) |
|
645 | 641 | else: |
|
646 | 642 | return defer.fail(AttributeError(cmd.remoteMethod)) |
|
647 | 643 | |
|
648 | 644 | def _flushQueue(self): |
|
649 | 645 | """Pop next command in queue and run it.""" |
|
650 | 646 | |
|
651 | 647 | if len(self.queued) > 0: |
|
652 | 648 | self.currentCommand = self.queued.pop(0) |
|
653 | 649 | self.runCurrentCommand() |
|
654 | 650 | |
|
655 | 651 | def saveResult(self, result): |
|
656 | 652 | """Put the result in the history.""" |
|
657 | 653 | self.history[result['number']] = result |
|
658 | 654 | return result |
|
659 | 655 | |
|
660 | 656 | def finishCommand(self, result): |
|
661 | 657 | """Finish currrent command.""" |
|
662 | 658 | |
|
663 | 659 | # The order of these commands is absolutely critical. |
|
664 | 660 | self.currentCommand.handleResult(result) |
|
665 | 661 | self.currentCommand.finished = True |
|
666 | 662 | self._flushQueue() |
|
667 | 663 | return result |
|
668 | 664 | |
|
669 | 665 | def abortCommand(self, reason): |
|
670 | 666 | """Abort current command. |
|
671 | 667 | |
|
672 | 668 | This eats the Failure but first passes it onto the Deferred that the |
|
673 | 669 | user has. |
|
674 | 670 | |
|
675 | 671 | It also clear out the queue so subsequence commands don't run. |
|
676 | 672 | """ |
|
677 | 673 | |
|
678 | 674 | # The order of these 3 commands is absolutely critical. The currentCommand |
|
679 | 675 | # must first be marked as finished BEFORE the queue is cleared and before |
|
680 | 676 | # the current command is sent the failure. |
|
681 | 677 | # Also, the queue must be cleared BEFORE the current command is sent the Failure |
|
682 | 678 | # otherwise the errback chain could trigger new commands to be added to the |
|
683 | 679 | # queue before we clear it. We should clear ONLY the commands that were in |
|
684 | 680 | # the queue when the error occured. |
|
685 | 681 | self.currentCommand.finished = True |
|
686 | 682 | s = "%r %r %r" % (self.currentCommand.remoteMethod, self.currentCommand.args, self.currentCommand.kwargs) |
|
687 | 683 | self.clear_queue(msg=s) |
|
688 | 684 | self.currentCommand.handleError(reason) |
|
689 | 685 | |
|
690 | 686 | return None |
|
691 | 687 | |
|
692 | 688 | #--------------------------------------------------------------------------- |
|
693 | 689 | # IEngineCore methods |
|
694 | 690 | #--------------------------------------------------------------------------- |
|
695 | 691 | |
|
696 | 692 | @queue |
|
697 | 693 | def execute(self, lines): |
|
698 | 694 | pass |
|
699 | 695 | |
|
700 | 696 | @queue |
|
701 | 697 | def push(self, namespace): |
|
702 | 698 | pass |
|
703 | 699 | |
|
704 | 700 | @queue |
|
705 | 701 | def pull(self, keys): |
|
706 | 702 | pass |
|
707 | 703 | |
|
708 | 704 | @queue |
|
709 | 705 | def push_function(self, namespace): |
|
710 | 706 | pass |
|
711 | 707 | |
|
712 | 708 | @queue |
|
713 | 709 | def pull_function(self, keys): |
|
714 | 710 | pass |
|
715 | 711 | |
|
716 | 712 | def get_result(self, i=None): |
|
717 | 713 | if i is None: |
|
718 | 714 | i = max(self.history.keys()+[None]) |
|
719 | 715 | |
|
720 | 716 | cmd = self.history.get(i, None) |
|
721 | 717 | # Uncomment this line to disable chaching of results |
|
722 | 718 | #cmd = None |
|
723 | 719 | if cmd is None: |
|
724 | 720 | return self.submitCommand(Command('get_result', i)) |
|
725 | 721 | else: |
|
726 | 722 | return defer.succeed(cmd) |
|
727 | 723 | |
|
728 | 724 | def reset(self): |
|
729 | 725 | self.clear_queue() |
|
730 | 726 | self.history = {} # reset the cache - I am not sure we should do this |
|
731 | 727 | return self.submitCommand(Command('reset')) |
|
732 | 728 | |
|
733 | 729 | def kill(self): |
|
734 | 730 | self.clear_queue() |
|
735 | 731 | return self.submitCommand(Command('kill')) |
|
736 | 732 | |
|
737 | 733 | @queue |
|
738 | 734 | def keys(self): |
|
739 | 735 | pass |
|
740 | 736 | |
|
741 | 737 | #--------------------------------------------------------------------------- |
|
742 | 738 | # IEngineSerialized methods |
|
743 | 739 | #--------------------------------------------------------------------------- |
|
744 | 740 | |
|
745 | 741 | @queue |
|
746 | 742 | def push_serialized(self, namespace): |
|
747 | 743 | pass |
|
748 | 744 | |
|
749 | 745 | @queue |
|
750 | 746 | def pull_serialized(self, keys): |
|
751 | 747 | pass |
|
752 | 748 | |
|
753 | 749 | #--------------------------------------------------------------------------- |
|
754 | 750 | # IEngineProperties methods |
|
755 | 751 | #--------------------------------------------------------------------------- |
|
756 | 752 | |
|
757 | 753 | @queue |
|
758 | 754 | def set_properties(self, namespace): |
|
759 | 755 | pass |
|
760 | 756 | |
|
761 | 757 | @queue |
|
762 | 758 | def get_properties(self, keys=None): |
|
763 | 759 | pass |
|
764 | 760 | |
|
765 | 761 | @queue |
|
766 | 762 | def del_properties(self, keys): |
|
767 | 763 | pass |
|
768 | 764 | |
|
769 | 765 | @queue |
|
770 | 766 | def has_properties(self, keys): |
|
771 | 767 | pass |
|
772 | 768 | |
|
773 | 769 | @queue |
|
774 | 770 | def clear_properties(self): |
|
775 | 771 | pass |
|
776 | 772 | |
|
777 | 773 | #--------------------------------------------------------------------------- |
|
778 | 774 | # IQueuedEngine methods |
|
779 | 775 | #--------------------------------------------------------------------------- |
|
780 | 776 | |
|
781 | 777 | def clear_queue(self, msg=''): |
|
782 | 778 | """Clear the queue, but doesn't cancel the currently running commmand.""" |
|
783 | 779 | |
|
784 | 780 | for cmd in self.queued: |
|
785 | 781 | cmd.deferred.errback(failure.Failure(error.QueueCleared(msg))) |
|
786 | 782 | self.queued = [] |
|
787 | 783 | return defer.succeed(None) |
|
788 | 784 | |
|
789 | 785 | def queue_status(self): |
|
790 | 786 | if self.currentCommand is not None: |
|
791 | 787 | if self.currentCommand.finished: |
|
792 | 788 | pending = repr(None) |
|
793 | 789 | else: |
|
794 | 790 | pending = repr(self.currentCommand) |
|
795 | 791 | else: |
|
796 | 792 | pending = repr(None) |
|
797 | 793 | dikt = {'queue':map(repr,self.queued), 'pending':pending} |
|
798 | 794 | return defer.succeed(dikt) |
|
799 | 795 | |
|
800 | 796 | def register_failure_observer(self, obs): |
|
801 | 797 | self.failureObservers.append(obs) |
|
802 | 798 | |
|
803 | 799 | def unregister_failure_observer(self, obs): |
|
804 | 800 | self.failureObservers.remove(obs) |
|
805 | 801 | |
|
806 | 802 | |
|
807 | 803 | # Now register QueuedEngine as an adpater class that makes an IEngineBase into a |
|
808 | 804 | # IEngineQueued. |
|
809 | 805 | components.registerAdapter(QueuedEngine, IEngineBase, IEngineQueued) |
|
810 | 806 | |
|
811 | 807 | |
|
812 | 808 | class Command(object): |
|
813 | 809 | """A command object that encapslates queued commands. |
|
814 | 810 | |
|
815 | 811 | This class basically keeps track of a command that has been queued |
|
816 | 812 | in a QueuedEngine. It manages the deferreds and hold the method to be called |
|
817 | 813 | and the arguments to that method. |
|
818 | 814 | """ |
|
819 | 815 | |
|
820 | 816 | |
|
821 | 817 | def __init__(self, remoteMethod, *args, **kwargs): |
|
822 | 818 | """Build a new Command object.""" |
|
823 | 819 | |
|
824 | 820 | self.remoteMethod = remoteMethod |
|
825 | 821 | self.args = args |
|
826 | 822 | self.kwargs = kwargs |
|
827 | 823 | self.finished = False |
|
828 | 824 | |
|
829 | 825 | def setDeferred(self, d): |
|
830 | 826 | """Sets the deferred attribute of the Command.""" |
|
831 | 827 | |
|
832 | 828 | self.deferred = d |
|
833 | 829 | |
|
834 | 830 | def __repr__(self): |
|
835 | 831 | if not self.args: |
|
836 | 832 | args = '' |
|
837 | 833 | else: |
|
838 | 834 | args = str(self.args)[1:-2] #cut off (...,) |
|
839 | 835 | for k,v in self.kwargs.iteritems(): |
|
840 | 836 | if args: |
|
841 | 837 | args += ', ' |
|
842 | 838 | args += '%s=%r' %(k,v) |
|
843 | 839 | return "%s(%s)" %(self.remoteMethod, args) |
|
844 | 840 | |
|
845 | 841 | def handleResult(self, result): |
|
846 | 842 | """When the result is ready, relay it to self.deferred.""" |
|
847 | 843 | |
|
848 | 844 | self.deferred.callback(result) |
|
849 | 845 | |
|
850 | 846 | def handleError(self, reason): |
|
851 | 847 | """When an error has occured, relay it to self.deferred.""" |
|
852 | 848 | |
|
853 | 849 | self.deferred.errback(reason) |
|
854 | 850 | |
|
855 | 851 | class ThreadedEngineService(EngineService): |
|
856 | 852 | """An EngineService subclass that defers execute commands to a separate |
|
857 | 853 | thread. |
|
858 | 854 | |
|
859 | 855 | ThreadedEngineService uses twisted.internet.threads.deferToThread to |
|
860 | 856 | defer execute requests to a separate thread. GUI frontends may want to |
|
861 | 857 | use ThreadedEngineService as the engine in an |
|
862 | 858 | IPython.frontend.frontendbase.FrontEndBase subclass to prevent |
|
863 | 859 | block execution from blocking the GUI thread. |
|
864 | 860 | """ |
|
865 | 861 | |
|
866 | 862 | zi.implements(IEngineBase) |
|
867 | 863 | |
|
868 | 864 | def __init__(self, shellClass=Interpreter, mpi=None): |
|
869 | 865 | EngineService.__init__(self, shellClass, mpi) |
|
870 | 866 | |
|
871 | 867 | def wrapped_execute(self, msg, lines): |
|
872 | 868 | """Wrap self.shell.execute to add extra information to tracebacks""" |
|
873 | 869 | |
|
874 | 870 | try: |
|
875 | 871 | result = self.shell.execute(lines) |
|
876 | 872 | except Exception,e: |
|
877 | 873 | # This gives the following: |
|
878 | 874 | # et=exception class |
|
879 | 875 | # ev=exception class instance |
|
880 | 876 | # tb=traceback object |
|
881 | 877 | et,ev,tb = sys.exc_info() |
|
882 | 878 | # This call adds attributes to the exception value |
|
883 | 879 | et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg) |
|
884 | 880 | # Add another attribute |
|
885 | 881 | |
|
886 | 882 | # Create a new exception with the new attributes |
|
887 | 883 | e = et(ev._ipython_traceback_text) |
|
888 | 884 | e._ipython_engine_info = msg |
|
889 | 885 | |
|
890 | 886 | # Re-raise |
|
891 | 887 | raise e |
|
892 | 888 | |
|
893 | 889 | return result |
|
894 | 890 | |
|
895 | 891 | |
|
896 | 892 | def execute(self, lines): |
|
897 | 893 | # Only import this if we are going to use this class |
|
898 | 894 | from twisted.internet import threads |
|
899 | 895 | |
|
900 | 896 | msg = {'engineid':self.id, |
|
901 | 897 | 'method':'execute', |
|
902 | 898 | 'args':[lines]} |
|
903 | 899 | |
|
904 | 900 | d = threads.deferToThread(self.wrapped_execute, msg, lines) |
|
905 | 901 | d.addCallback(self.addIDToResult) |
|
906 | 902 | return d |
@@ -1,202 +1,205 b'' | |||
|
1 | 1 | # encoding: utf-8 |
|
2 | 2 | |
|
3 | 3 | """Classes and functions for kernel related errors and exceptions.""" |
|
4 | 4 | |
|
5 | 5 | __docformat__ = "restructuredtext en" |
|
6 | 6 | |
|
7 | # Tell nose to skip this module | |
|
8 | __test__ = {} | |
|
9 | ||
|
7 | 10 | #------------------------------------------------------------------------------- |
|
8 | 11 | # Copyright (C) 2008 The IPython Development Team |
|
9 | 12 | # |
|
10 | 13 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | 14 | # the file COPYING, distributed as part of this software. |
|
12 | 15 | #------------------------------------------------------------------------------- |
|
13 | 16 | |
|
14 | 17 | #------------------------------------------------------------------------------- |
|
15 | 18 | # Imports |
|
16 | 19 | #------------------------------------------------------------------------------- |
|
20 | from twisted.python import failure | |
|
17 | 21 | |
|
18 | 22 | from IPython.kernel.core import error |
|
19 | from twisted.python import failure | |
|
20 | 23 | |
|
21 | 24 | #------------------------------------------------------------------------------- |
|
22 | 25 | # Error classes |
|
23 | 26 | #------------------------------------------------------------------------------- |
|
24 | 27 | |
|
25 | 28 | class KernelError(error.IPythonError): |
|
26 | 29 | pass |
|
27 | 30 | |
|
28 | 31 | class NotDefined(KernelError): |
|
29 | 32 | def __init__(self, name): |
|
30 | 33 | self.name = name |
|
31 | 34 | self.args = (name,) |
|
32 | 35 | |
|
33 | 36 | def __repr__(self): |
|
34 | 37 | return '<NotDefined: %s>' % self.name |
|
35 | 38 | |
|
36 | 39 | __str__ = __repr__ |
|
37 | 40 | |
|
38 | 41 | class QueueCleared(KernelError): |
|
39 | 42 | pass |
|
40 | 43 | |
|
41 | 44 | class IdInUse(KernelError): |
|
42 | 45 | pass |
|
43 | 46 | |
|
44 | 47 | class ProtocolError(KernelError): |
|
45 | 48 | pass |
|
46 | 49 | |
|
47 | 50 | class ConnectionError(KernelError): |
|
48 | 51 | pass |
|
49 | 52 | |
|
50 | 53 | class InvalidEngineID(KernelError): |
|
51 | 54 | pass |
|
52 | 55 | |
|
53 | 56 | class NoEnginesRegistered(KernelError): |
|
54 | 57 | pass |
|
55 | 58 | |
|
56 | 59 | class InvalidClientID(KernelError): |
|
57 | 60 | pass |
|
58 | 61 | |
|
59 | 62 | class InvalidDeferredID(KernelError): |
|
60 | 63 | pass |
|
61 | 64 | |
|
62 | 65 | class SerializationError(KernelError): |
|
63 | 66 | pass |
|
64 | 67 | |
|
65 | 68 | class MessageSizeError(KernelError): |
|
66 | 69 | pass |
|
67 | 70 | |
|
68 | 71 | class PBMessageSizeError(MessageSizeError): |
|
69 | 72 | pass |
|
70 | 73 | |
|
71 | 74 | class ResultNotCompleted(KernelError): |
|
72 | 75 | pass |
|
73 | 76 | |
|
74 | 77 | class ResultAlreadyRetrieved(KernelError): |
|
75 | 78 | pass |
|
76 | 79 | |
|
77 | 80 | class ClientError(KernelError): |
|
78 | 81 | pass |
|
79 | 82 | |
|
80 | 83 | class TaskAborted(KernelError): |
|
81 | 84 | pass |
|
82 | 85 | |
|
83 | 86 | class TaskTimeout(KernelError): |
|
84 | 87 | pass |
|
85 | 88 | |
|
86 | 89 | class NotAPendingResult(KernelError): |
|
87 | 90 | pass |
|
88 | 91 | |
|
89 | 92 | class UnpickleableException(KernelError): |
|
90 | 93 | pass |
|
91 | 94 | |
|
92 | 95 | class AbortedPendingDeferredError(KernelError): |
|
93 | 96 | pass |
|
94 | 97 | |
|
95 | 98 | class InvalidProperty(KernelError): |
|
96 | 99 | pass |
|
97 | 100 | |
|
98 | 101 | class MissingBlockArgument(KernelError): |
|
99 | 102 | pass |
|
100 | 103 | |
|
101 | 104 | class StopLocalExecution(KernelError): |
|
102 | 105 | pass |
|
103 | 106 | |
|
104 | 107 | class SecurityError(KernelError): |
|
105 | 108 | pass |
|
106 | 109 | |
|
107 | 110 | class FileTimeoutError(KernelError): |
|
108 | 111 | pass |
|
109 | 112 | |
|
110 | 113 | class TaskRejectError(KernelError): |
|
111 | 114 | """Exception to raise when a task should be rejected by an engine. |
|
112 | 115 | |
|
113 | 116 | This exception can be used to allow a task running on an engine to test |
|
114 | 117 | if the engine (or the user's namespace on the engine) has the needed |
|
115 | 118 | task dependencies. If not, the task should raise this exception. For |
|
116 | 119 | the task to be retried on another engine, the task should be created |
|
117 | 120 | with the `retries` argument > 1. |
|
118 | 121 | |
|
119 | 122 | The advantage of this approach over our older properties system is that |
|
120 | 123 | tasks have full access to the user's namespace on the engines and the |
|
121 | 124 | properties don't have to be managed or tested by the controller. |
|
122 | 125 | """ |
|
123 | 126 | |
|
124 | 127 | class CompositeError(KernelError): |
|
125 | 128 | def __init__(self, message, elist): |
|
126 | 129 | Exception.__init__(self, *(message, elist)) |
|
127 | 130 | self.message = message |
|
128 | 131 | self.elist = elist |
|
129 | 132 | |
|
130 | 133 | def _get_engine_str(self, ev): |
|
131 | 134 | try: |
|
132 | 135 | ei = ev._ipython_engine_info |
|
133 | 136 | except AttributeError: |
|
134 | 137 | return '[Engine Exception]' |
|
135 | 138 | else: |
|
136 | 139 | return '[%i:%s]: ' % (ei['engineid'], ei['method']) |
|
137 | 140 | |
|
138 | 141 | def _get_traceback(self, ev): |
|
139 | 142 | try: |
|
140 | 143 | tb = ev._ipython_traceback_text |
|
141 | 144 | except AttributeError: |
|
142 | 145 | return 'No traceback available' |
|
143 | 146 | else: |
|
144 | 147 | return tb |
|
145 | 148 | |
|
146 | 149 | def __str__(self): |
|
147 | 150 | s = str(self.message) |
|
148 | 151 | for et, ev, etb in self.elist: |
|
149 | 152 | engine_str = self._get_engine_str(ev) |
|
150 | 153 | s = s + '\n' + engine_str + str(et.__name__) + ': ' + str(ev) |
|
151 | 154 | return s |
|
152 | 155 | |
|
153 | 156 | def print_tracebacks(self, excid=None): |
|
154 | 157 | if excid is None: |
|
155 | 158 | for (et,ev,etb) in self.elist: |
|
156 | 159 | print self._get_engine_str(ev) |
|
157 | 160 | print self._get_traceback(ev) |
|
158 | 161 | |
|
159 | 162 | else: |
|
160 | 163 | try: |
|
161 | 164 | et,ev,etb = self.elist[excid] |
|
162 | 165 | except: |
|
163 | 166 | raise IndexError("an exception with index %i does not exist"%excid) |
|
164 | 167 | else: |
|
165 | 168 | print self._get_engine_str(ev) |
|
166 | 169 | print self._get_traceback(ev) |
|
167 | 170 | |
|
168 | 171 | def raise_exception(self, excid=0): |
|
169 | 172 | try: |
|
170 | 173 | et,ev,etb = self.elist[excid] |
|
171 | 174 | except: |
|
172 | 175 | raise IndexError("an exception with index %i does not exist"%excid) |
|
173 | 176 | else: |
|
174 | 177 | raise et, ev, etb |
|
175 | 178 | |
|
176 | 179 | def collect_exceptions(rlist, method): |
|
177 | 180 | elist = [] |
|
178 | 181 | for r in rlist: |
|
179 | 182 | if isinstance(r, failure.Failure): |
|
180 | 183 | r.cleanFailure() |
|
181 | 184 | et, ev, etb = r.type, r.value, r.tb |
|
182 | 185 | # Sometimes we could have CompositeError in our list. Just take |
|
183 | 186 | # the errors out of them and put them in our new list. This |
|
184 | 187 | # has the effect of flattening lists of CompositeErrors into one |
|
185 | 188 | # CompositeError |
|
186 | 189 | if et==CompositeError: |
|
187 | 190 | for e in ev.elist: |
|
188 | 191 | elist.append(e) |
|
189 | 192 | else: |
|
190 | 193 | elist.append((et, ev, etb)) |
|
191 | 194 | if len(elist)==0: |
|
192 | 195 | return rlist |
|
193 | 196 | else: |
|
194 | 197 | msg = "one or more exceptions from call to method: %s" % (method) |
|
195 | 198 | # This silliness is needed so the debugger has access to the exception |
|
196 | 199 | # instance (e in this case) |
|
197 | 200 | try: |
|
198 | 201 | raise CompositeError(msg, elist) |
|
199 | 202 | except CompositeError, e: |
|
200 | 203 | raise e |
|
201 | 204 | |
|
202 | 205 |
@@ -1,163 +1,170 b'' | |||
|
1 | 1 | # encoding: utf-8 |
|
2 | 2 | # -*- test-case-name: IPython.kernel.test.test_newserialized -*- |
|
3 | 3 | |
|
4 | 4 | """Refactored serialization classes and interfaces.""" |
|
5 | 5 | |
|
6 | 6 | __docformat__ = "restructuredtext en" |
|
7 | 7 | |
|
8 | # Tell nose to skip this module | |
|
9 | __test__ = {} | |
|
10 | ||
|
8 | 11 | #------------------------------------------------------------------------------- |
|
9 | 12 | # Copyright (C) 2008 The IPython Development Team |
|
10 | 13 | # |
|
11 | 14 | # Distributed under the terms of the BSD License. The full license is in |
|
12 | 15 | # the file COPYING, distributed as part of this software. |
|
13 | 16 | #------------------------------------------------------------------------------- |
|
14 | 17 | |
|
15 | 18 | #------------------------------------------------------------------------------- |
|
16 | 19 | # Imports |
|
17 | 20 | #------------------------------------------------------------------------------- |
|
18 | 21 | |
|
19 | 22 | import cPickle as pickle |
|
20 | 23 | |
|
21 | from zope.interface import Interface, implements | |
|
22 | 24 | from twisted.python import components |
|
25 | from zope.interface import Interface, implements | |
|
23 | 26 | |
|
24 | 27 | try: |
|
25 | 28 | import numpy |
|
26 | 29 | except ImportError: |
|
27 | 30 | pass |
|
28 | 31 | |
|
29 | 32 | from IPython.kernel.error import SerializationError |
|
30 | 33 | |
|
34 | #----------------------------------------------------------------------------- | |
|
35 | # Classes and functions | |
|
36 | #----------------------------------------------------------------------------- | |
|
37 | ||
|
31 | 38 | class ISerialized(Interface): |
|
32 | 39 | |
|
33 | 40 | def getData(): |
|
34 | 41 | """""" |
|
35 | 42 | |
|
36 | 43 | def getDataSize(units=10.0**6): |
|
37 | 44 | """""" |
|
38 | 45 | |
|
39 | 46 | def getTypeDescriptor(): |
|
40 | 47 | """""" |
|
41 | 48 | |
|
42 | 49 | def getMetadata(): |
|
43 | 50 | """""" |
|
44 | 51 | |
|
45 | 52 | |
|
46 | 53 | class IUnSerialized(Interface): |
|
47 | 54 | |
|
48 | 55 | def getObject(): |
|
49 | 56 | """""" |
|
50 | 57 | |
|
51 | 58 | class Serialized(object): |
|
52 | 59 | |
|
53 | 60 | implements(ISerialized) |
|
54 | 61 | |
|
55 | 62 | def __init__(self, data, typeDescriptor, metadata={}): |
|
56 | 63 | self.data = data |
|
57 | 64 | self.typeDescriptor = typeDescriptor |
|
58 | 65 | self.metadata = metadata |
|
59 | 66 | |
|
60 | 67 | def getData(self): |
|
61 | 68 | return self.data |
|
62 | 69 | |
|
63 | 70 | def getDataSize(self, units=10.0**6): |
|
64 | 71 | return len(self.data)/units |
|
65 | 72 | |
|
66 | 73 | def getTypeDescriptor(self): |
|
67 | 74 | return self.typeDescriptor |
|
68 | 75 | |
|
69 | 76 | def getMetadata(self): |
|
70 | 77 | return self.metadata |
|
71 | 78 | |
|
72 | 79 | |
|
73 | 80 | class UnSerialized(object): |
|
74 | 81 | |
|
75 | 82 | implements(IUnSerialized) |
|
76 | 83 | |
|
77 | 84 | def __init__(self, obj): |
|
78 | 85 | self.obj = obj |
|
79 | 86 | |
|
80 | 87 | def getObject(self): |
|
81 | 88 | return self.obj |
|
82 | 89 | |
|
83 | 90 | |
|
84 | 91 | class SerializeIt(object): |
|
85 | 92 | |
|
86 | 93 | implements(ISerialized) |
|
87 | 94 | |
|
88 | 95 | def __init__(self, unSerialized): |
|
89 | 96 | self.data = None |
|
90 | 97 | self.obj = unSerialized.getObject() |
|
91 | 98 | if globals().has_key('numpy'): |
|
92 | 99 | if isinstance(self.obj, numpy.ndarray): |
|
93 | 100 | if len(self.obj) == 0: # length 0 arrays can't be reconstructed |
|
94 | 101 | raise SerializationError("You cannot send a length 0 array") |
|
95 | 102 | self.obj = numpy.ascontiguousarray(self.obj, dtype=None) |
|
96 | 103 | self.typeDescriptor = 'ndarray' |
|
97 | 104 | self.metadata = {'shape':self.obj.shape, |
|
98 | 105 | 'dtype':self.obj.dtype.str} |
|
99 | 106 | else: |
|
100 | 107 | self.typeDescriptor = 'pickle' |
|
101 | 108 | self.metadata = {} |
|
102 | 109 | else: |
|
103 | 110 | self.typeDescriptor = 'pickle' |
|
104 | 111 | self.metadata = {} |
|
105 | 112 | self._generateData() |
|
106 | 113 | |
|
107 | 114 | def _generateData(self): |
|
108 | 115 | if self.typeDescriptor == 'ndarray': |
|
109 | 116 | self.data = numpy.getbuffer(self.obj) |
|
110 | 117 | elif self.typeDescriptor == 'pickle': |
|
111 | 118 | self.data = pickle.dumps(self.obj, 2) |
|
112 | 119 | else: |
|
113 | 120 | raise SerializationError("Really wierd serialization error.") |
|
114 | 121 | del self.obj |
|
115 | 122 | |
|
116 | 123 | def getData(self): |
|
117 | 124 | return self.data |
|
118 | 125 | |
|
119 | 126 | def getDataSize(self, units=10.0**6): |
|
120 | 127 | return len(self.data)/units |
|
121 | 128 | |
|
122 | 129 | def getTypeDescriptor(self): |
|
123 | 130 | return self.typeDescriptor |
|
124 | 131 | |
|
125 | 132 | def getMetadata(self): |
|
126 | 133 | return self.metadata |
|
127 | 134 | |
|
128 | 135 | |
|
129 | 136 | class UnSerializeIt(UnSerialized): |
|
130 | 137 | |
|
131 | 138 | implements(IUnSerialized) |
|
132 | 139 | |
|
133 | 140 | def __init__(self, serialized): |
|
134 | 141 | self.serialized = serialized |
|
135 | 142 | |
|
136 | 143 | def getObject(self): |
|
137 | 144 | typeDescriptor = self.serialized.getTypeDescriptor() |
|
138 | 145 | if globals().has_key('numpy'): |
|
139 | 146 | if typeDescriptor == 'ndarray': |
|
140 | 147 | result = numpy.frombuffer(self.serialized.getData(), dtype = self.serialized.metadata['dtype']) |
|
141 | 148 | result.shape = self.serialized.metadata['shape'] |
|
142 | 149 | # This is a hack to make the array writable. We are working with |
|
143 | 150 | # the numpy folks to address this issue. |
|
144 | 151 | result = result.copy() |
|
145 | 152 | elif typeDescriptor == 'pickle': |
|
146 | 153 | result = pickle.loads(self.serialized.getData()) |
|
147 | 154 | else: |
|
148 | 155 | raise SerializationError("Really wierd serialization error.") |
|
149 | 156 | elif typeDescriptor == 'pickle': |
|
150 | 157 | result = pickle.loads(self.serialized.getData()) |
|
151 | 158 | else: |
|
152 | 159 | raise SerializationError("Really wierd serialization error.") |
|
153 | 160 | return result |
|
154 | 161 | |
|
155 | 162 | components.registerAdapter(UnSerializeIt, ISerialized, IUnSerialized) |
|
156 | 163 | |
|
157 | 164 | components.registerAdapter(SerializeIt, IUnSerialized, ISerialized) |
|
158 | 165 | |
|
159 | 166 | def serialize(obj): |
|
160 | 167 | return ISerialized(UnSerialized(obj)) |
|
161 | 168 | |
|
162 | 169 | def unserialize(serialized): |
|
163 | 170 | return IUnSerialized(serialized).getObject() |
@@ -1,326 +1,346 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """IPython Test Suite Runner. |
|
3 | 3 | |
|
4 | 4 | This module provides a main entry point to a user script to test IPython |
|
5 | 5 | itself from the command line. There are two ways of running this script: |
|
6 | 6 | |
|
7 | 7 | 1. With the syntax `iptest all`. This runs our entire test suite by |
|
8 | 8 | calling this script (with different arguments) or trial recursively. This |
|
9 | 9 | causes modules and package to be tested in different processes, using nose |
|
10 | 10 | or trial where appropriate. |
|
11 | 11 | 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form |
|
12 | 12 | the script simply calls nose, but with special command line flags and |
|
13 | 13 | plugins loaded. |
|
14 | 14 | |
|
15 | 15 | For now, this script requires that both nose and twisted are installed. This |
|
16 | 16 | will change in the future. |
|
17 | 17 | """ |
|
18 | 18 | |
|
19 | 19 | #----------------------------------------------------------------------------- |
|
20 | 20 | # Module imports |
|
21 | 21 | #----------------------------------------------------------------------------- |
|
22 | 22 | |
|
23 | 23 | import os |
|
24 | 24 | import os.path as path |
|
25 | 25 | import sys |
|
26 | 26 | import subprocess |
|
27 | 27 | import tempfile |
|
28 | 28 | import time |
|
29 | 29 | import warnings |
|
30 | 30 | |
|
31 | 31 | import nose.plugins.builtin |
|
32 | 32 | from nose.core import TestProgram |
|
33 | 33 | |
|
34 | 34 | from IPython.platutils import find_cmd |
|
35 | 35 | from IPython.testing.plugin.ipdoctest import IPythonDoctest |
|
36 | 36 | |
|
37 | 37 | pjoin = path.join |
|
38 | 38 | |
|
39 | 39 | #----------------------------------------------------------------------------- |
|
40 | 40 | # Logic for skipping doctests |
|
41 | 41 | #----------------------------------------------------------------------------- |
|
42 | 42 | |
|
43 | 43 | def test_for(mod): |
|
44 | 44 | """Test to see if mod is importable.""" |
|
45 | 45 | try: |
|
46 | 46 | __import__(mod) |
|
47 | 47 | except ImportError: |
|
48 | 48 | return False |
|
49 | 49 | else: |
|
50 | 50 | return True |
|
51 | 51 | |
|
52 | 52 | have_curses = test_for('_curses') |
|
53 | 53 | have_wx = test_for('wx') |
|
54 | 54 | have_wx_aui = test_for('wx.aui') |
|
55 | 55 | have_zi = test_for('zope.interface') |
|
56 | 56 | have_twisted = test_for('twisted') |
|
57 | 57 | have_foolscap = test_for('foolscap') |
|
58 | 58 | have_objc = test_for('objc') |
|
59 | 59 | have_pexpect = test_for('pexpect') |
|
60 | 60 | |
|
61 | 61 | # For the IPythonDoctest plugin, we need to exclude certain patterns that cause |
|
62 | 62 | # testing problems. We should strive to minimize the number of skipped |
|
63 | 63 | # modules, since this means untested code. As the testing machinery |
|
64 | 64 | # solidifies, this list should eventually become empty. |
|
65 | 65 | EXCLUDE = [pjoin('IPython', 'external'), |
|
66 | 66 | pjoin('IPython', 'frontend', 'process', 'winprocess.py'), |
|
67 | 67 | pjoin('IPython_doctest_plugin'), |
|
68 | 68 | pjoin('IPython', 'Gnuplot'), |
|
69 | 69 | pjoin('IPython', 'Extensions', 'ipy_'), |
|
70 | 70 | pjoin('IPython', 'Extensions', 'PhysicalQInput'), |
|
71 | 71 | pjoin('IPython', 'Extensions', 'PhysicalQInteractive'), |
|
72 | 72 | pjoin('IPython', 'Extensions', 'InterpreterPasteInput'), |
|
73 | 73 | pjoin('IPython', 'Extensions', 'scitedirector'), |
|
74 | 74 | pjoin('IPython', 'Extensions', 'numeric_formats'), |
|
75 | 75 | pjoin('IPython', 'testing', 'attic'), |
|
76 | 76 | pjoin('IPython', 'testing', 'tutils'), |
|
77 | 77 | pjoin('IPython', 'testing', 'tools'), |
|
78 | 78 | pjoin('IPython', 'testing', 'mkdoctests'), |
|
79 | 79 | ] |
|
80 | 80 | |
|
81 | 81 | if not have_wx: |
|
82 | 82 | EXCLUDE.append(pjoin('IPython', 'Extensions', 'igrid')) |
|
83 | 83 | EXCLUDE.append(pjoin('IPython', 'gui')) |
|
84 | 84 | EXCLUDE.append(pjoin('IPython', 'frontend', 'wx')) |
|
85 | 85 | |
|
86 | 86 | if not have_wx_aui: |
|
87 | 87 | EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython')) |
|
88 | 88 | |
|
89 | 89 | if not have_objc: |
|
90 | 90 | EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa')) |
|
91 | 91 | |
|
92 | 92 | if not have_curses: |
|
93 | 93 | EXCLUDE.append(pjoin('IPython', 'Extensions', 'ibrowse')) |
|
94 | 94 | |
|
95 | 95 | if not sys.platform == 'win32': |
|
96 | 96 | EXCLUDE.append(pjoin('IPython', 'platutils_win32')) |
|
97 | 97 | |
|
98 | 98 | # These have to be skipped on win32 because the use echo, rm, cd, etc. |
|
99 | 99 | # See ticket https://bugs.launchpad.net/bugs/366982 |
|
100 | 100 | if sys.platform == 'win32': |
|
101 | 101 | EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip')) |
|
102 | 102 | EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample')) |
|
103 | 103 | |
|
104 | 104 | if not os.name == 'posix': |
|
105 | 105 | EXCLUDE.append(pjoin('IPython', 'platutils_posix')) |
|
106 | 106 | |
|
107 | 107 | if not have_pexpect: |
|
108 | 108 | EXCLUDE.append(pjoin('IPython', 'irunner')) |
|
109 | 109 | |
|
110 | if not have_twisted: | |
|
111 | EXCLUDE.append(pjoin('IPython', 'frontend', 'asyncfrontendbase')) | |
|
112 | EXCLUDE.append(pjoin('IPython', 'frontend', 'prefilterfrontend')) | |
|
113 | EXCLUDE.append(pjoin('IPython', 'frontend', 'frontendbase')) | |
|
114 | EXCLUDE.append(pjoin('IPython', 'frontend', 'linefrontendbase')) | |
|
115 | EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', 'test_linefrontend')) | |
|
116 | EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', 'test_frontendbase')) | |
|
117 | EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', | |
|
118 | 'test_prefilterfrontend')) | |
|
119 | EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', | |
|
120 | 'test_asyncfrontendbase')) | |
|
121 | EXCLUDE.append(pjoin('IPython', 'kernel', 'error')) | |
|
122 | EXCLUDE.append(pjoin('IPython', 'testing', 'parametric')) | |
|
123 | EXCLUDE.append(pjoin('IPython', 'testing', 'util')) | |
|
124 | EXCLUDE.append(pjoin('IPython', 'testing', 'tests', 'test_decorators_trial')) | |
|
125 | ||
|
126 | ||
|
110 | 127 | # This is needed for the reg-exp to match on win32 in the ipdoctest plugin. |
|
111 | 128 | if sys.platform == 'win32': |
|
112 | 129 | EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE] |
|
113 | 130 | |
|
114 | 131 | |
|
115 | 132 | #----------------------------------------------------------------------------- |
|
116 | 133 | # Functions and classes |
|
117 | 134 | #----------------------------------------------------------------------------- |
|
118 | 135 | |
|
119 | 136 | def run_iptest(): |
|
120 | 137 | """Run the IPython test suite using nose. |
|
121 | 138 | |
|
122 | 139 | This function is called when this script is **not** called with the form |
|
123 | 140 | `iptest all`. It simply calls nose with appropriate command line flags |
|
124 | 141 | and accepts all of the standard nose arguments. |
|
125 | 142 | """ |
|
126 | 143 | |
|
127 | 144 | warnings.filterwarnings('ignore', |
|
128 | 145 | 'This will be removed soon. Use IPython.testing.util instead') |
|
129 | 146 | |
|
130 | 147 | argv = sys.argv + [ |
|
131 | 148 | # Loading ipdoctest causes problems with Twisted. |
|
132 | 149 | # I am removing this as a temporary fix to get the |
|
133 | 150 | # test suite back into working shape. Our nose |
|
134 | 151 | # plugin needs to be gone through with a fine |
|
135 | 152 | # toothed comb to find what is causing the problem. |
|
136 | 153 | '--with-ipdoctest', |
|
137 | 154 | '--ipdoctest-tests','--ipdoctest-extension=txt', |
|
138 | 155 | '--detailed-errors', |
|
139 | 156 | |
|
140 | 157 | # We add --exe because of setuptools' imbecility (it |
|
141 | 158 | # blindly does chmod +x on ALL files). Nose does the |
|
142 | 159 | # right thing and it tries to avoid executables, |
|
143 | 160 | # setuptools unfortunately forces our hand here. This |
|
144 | 161 | # has been discussed on the distutils list and the |
|
145 | 162 | # setuptools devs refuse to fix this problem! |
|
146 | 163 | '--exe', |
|
147 | 164 | ] |
|
148 | 165 | |
|
149 | 166 | # Detect if any tests were required by explicitly calling an IPython |
|
150 | 167 | # submodule or giving a specific path |
|
151 | 168 | has_tests = False |
|
152 | 169 | for arg in sys.argv: |
|
153 | 170 | if 'IPython' in arg or arg.endswith('.py') or \ |
|
154 | 171 | (':' in arg and '.py' in arg): |
|
155 | 172 | has_tests = True |
|
156 | 173 | break |
|
157 | 174 | |
|
158 | 175 | # If nothing was specifically requested, test full IPython |
|
159 | 176 | if not has_tests: |
|
160 | 177 | argv.append('IPython') |
|
161 | 178 | |
|
162 | 179 | # Construct list of plugins, omitting the existing doctest plugin, which |
|
163 | 180 | # ours replaces (and extends). |
|
164 | 181 | plugins = [IPythonDoctest(EXCLUDE)] |
|
165 | 182 | for p in nose.plugins.builtin.plugins: |
|
166 | 183 | plug = p() |
|
167 | 184 | if plug.name == 'doctest': |
|
168 | 185 | continue |
|
169 | 186 | |
|
170 | 187 | #print '*** adding plugin:',plug.name # dbg |
|
171 | 188 | plugins.append(plug) |
|
172 | 189 | |
|
173 | 190 | TestProgram(argv=argv,plugins=plugins) |
|
174 | 191 | |
|
175 | 192 | |
|
176 | 193 | class IPTester(object): |
|
177 | 194 | """Call that calls iptest or trial in a subprocess. |
|
178 | 195 | """ |
|
179 | 196 | def __init__(self,runner='iptest',params=None): |
|
180 | 197 | """ """ |
|
181 | 198 | if runner == 'iptest': |
|
182 | 199 | self.runner = ['iptest','-v'] |
|
183 | 200 | else: |
|
184 | 201 | self.runner = [find_cmd('trial')] |
|
185 | 202 | if params is None: |
|
186 | 203 | params = [] |
|
187 | 204 | if isinstance(params,str): |
|
188 | 205 | params = [params] |
|
189 | 206 | self.params = params |
|
190 | 207 | |
|
191 | 208 | # Assemble call |
|
192 | 209 | self.call_args = self.runner+self.params |
|
193 | 210 | |
|
194 | 211 | if sys.platform == 'win32': |
|
195 | 212 | def run(self): |
|
196 | 213 | """Run the stored commands""" |
|
197 | 214 | # On Windows, cd to temporary directory to run tests. Otherwise, |
|
198 | 215 | # Twisted's trial may not be able to execute 'trial IPython', since |
|
199 | 216 | # it will confuse the IPython module name with the ipython |
|
200 | 217 | # execution scripts, because the windows file system isn't case |
|
201 | 218 | # sensitive. |
|
202 | 219 | # We also use os.system instead of subprocess.call, because I was |
|
203 | 220 | # having problems with subprocess and I just don't know enough |
|
204 | 221 | # about win32 to debug this reliably. Os.system may be the 'old |
|
205 | 222 | # fashioned' way to do it, but it works just fine. If someone |
|
206 | 223 | # later can clean this up that's fine, as long as the tests run |
|
207 | 224 | # reliably in win32. |
|
208 | 225 | curdir = os.getcwd() |
|
209 | 226 | os.chdir(tempfile.gettempdir()) |
|
210 | 227 | stat = os.system(' '.join(self.call_args)) |
|
211 | 228 | os.chdir(curdir) |
|
212 | 229 | return stat |
|
213 | 230 | else: |
|
214 | 231 | def run(self): |
|
215 | 232 | """Run the stored commands""" |
|
216 | 233 | return subprocess.call(self.call_args) |
|
217 | 234 | |
|
218 | 235 | |
|
219 | 236 | def make_runners(): |
|
220 | 237 | """Define the modules and packages that need to be tested. |
|
221 | 238 | """ |
|
222 | 239 | |
|
223 | 240 | # This omits additional top-level modules that should not be doctested. |
|
224 | 241 | # XXX: Shell.py is also ommited because of a bug in the skip_doctest |
|
225 | 242 | # decorator. See ticket https://bugs.launchpad.net/bugs/366209 |
|
226 | 243 | top_mod = \ |
|
227 | 244 | ['background_jobs.py', 'ColorANSI.py', 'completer.py', 'ConfigLoader.py', |
|
228 | 245 | 'CrashHandler.py', 'Debugger.py', 'deep_reload.py', 'demo.py', |
|
229 | 246 | 'DPyGetOpt.py', 'dtutils.py', 'excolors.py', 'FakeModule.py', |
|
230 | 247 | 'generics.py', 'genutils.py', 'history.py', 'hooks.py', 'ipapi.py', |
|
231 | 248 | 'iplib.py', 'ipmaker.py', 'ipstruct.py', 'Itpl.py', |
|
232 | 249 | 'Logger.py', 'macro.py', 'Magic.py', 'OInspect.py', |
|
233 | 250 | 'OutputTrap.py', 'platutils.py', 'prefilter.py', 'Prompts.py', |
|
234 | 251 | 'PyColorize.py', 'Release.py', 'rlineimpl.py', 'shadowns.py', |
|
235 |
'shellglobals.py', 'strdispatch.py', |
|
|
252 | 'shellglobals.py', 'strdispatch.py', | |
|
236 | 253 | 'ultraTB.py', 'upgrade_dir.py', 'usage.py', 'wildcard.py', |
|
237 | 254 | # See note above for why this is skipped |
|
238 | 255 | # 'Shell.py', |
|
239 | 256 | 'winconsole.py'] |
|
240 | 257 | |
|
258 | if have_twisted: | |
|
259 | top_mod.append('twshell.py') | |
|
260 | ||
|
241 | 261 | if have_pexpect: |
|
242 | 262 | top_mod.append('irunner.py') |
|
243 | 263 | |
|
244 | 264 | if sys.platform == 'win32': |
|
245 | 265 | top_mod.append('platutils_win32.py') |
|
246 | 266 | elif os.name == 'posix': |
|
247 | 267 | top_mod.append('platutils_posix.py') |
|
248 | 268 | else: |
|
249 | 269 | top_mod.append('platutils_dummy.py') |
|
250 | 270 | |
|
251 | 271 | # These are tested by nose, so skip IPython.kernel |
|
252 | 272 | top_pack = ['config','Extensions','frontend', |
|
253 | 273 | 'testing','tests','tools','UserConfig'] |
|
254 | 274 | |
|
255 | 275 | if have_wx: |
|
256 | 276 | top_pack.append('gui') |
|
257 | 277 | |
|
258 | 278 | modules = ['IPython.%s' % m[:-3] for m in top_mod ] |
|
259 | 279 | packages = ['IPython.%s' % m for m in top_pack ] |
|
260 | 280 | |
|
261 | 281 | # Make runners |
|
262 | 282 | runners = dict(zip(top_pack, [IPTester(params=v) for v in packages])) |
|
263 | 283 | |
|
264 | 284 | # Test IPython.kernel using trial if twisted is installed |
|
265 | 285 | if have_zi and have_twisted and have_foolscap: |
|
266 | 286 | runners['trial'] = IPTester('trial',['IPython']) |
|
267 | 287 | |
|
268 | 288 | runners['modules'] = IPTester(params=modules) |
|
269 | 289 | |
|
270 | 290 | return runners |
|
271 | 291 | |
|
272 | 292 | |
|
273 | 293 | def run_iptestall(): |
|
274 | 294 | """Run the entire IPython test suite by calling nose and trial. |
|
275 | 295 | |
|
276 | 296 | This function constructs :class:`IPTester` instances for all IPython |
|
277 | 297 | modules and package and then runs each of them. This causes the modules |
|
278 | 298 | and packages of IPython to be tested each in their own subprocess using |
|
279 | 299 | nose or twisted.trial appropriately. |
|
280 | 300 | """ |
|
281 | 301 | runners = make_runners() |
|
282 | 302 | # Run all test runners, tracking execution time |
|
283 | 303 | failed = {} |
|
284 | 304 | t_start = time.time() |
|
285 | 305 | for name,runner in runners.iteritems(): |
|
286 | 306 | print '*'*77 |
|
287 | 307 | print 'IPython test group:',name |
|
288 | 308 | res = runner.run() |
|
289 | 309 | if res: |
|
290 | 310 | failed[name] = res |
|
291 | 311 | t_end = time.time() |
|
292 | 312 | t_tests = t_end - t_start |
|
293 | 313 | nrunners = len(runners) |
|
294 | 314 | nfail = len(failed) |
|
295 | 315 | # summarize results |
|
296 | 316 | |
|
297 | 317 | print '*'*77 |
|
298 | 318 | print 'Ran %s test groups in %.3fs' % (nrunners, t_tests) |
|
299 | 319 | |
|
300 | 320 | if not failed: |
|
301 | 321 | print 'OK' |
|
302 | 322 | else: |
|
303 | 323 | # If anything went wrong, point out what command to rerun manually to |
|
304 | 324 | # see the actual errors and individual summary |
|
305 | 325 | print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners) |
|
306 | 326 | for name in failed: |
|
307 | 327 | failed_runner = runners[name] |
|
308 | 328 | print '-'*40 |
|
309 | 329 | print 'Runner failed:',name |
|
310 | 330 | print 'You may wish to rerun this one individually, with:' |
|
311 | 331 | print ' '.join(failed_runner.call_args) |
|
312 | 332 | |
|
313 | 333 | |
|
314 | 334 | |
|
315 | 335 | def main(): |
|
316 | 336 | if len(sys.argv) == 1: |
|
317 | 337 | run_iptestall() |
|
318 | 338 | else: |
|
319 | 339 | if sys.argv[1] == 'all': |
|
320 | 340 | run_iptestall() |
|
321 | 341 | else: |
|
322 | 342 | run_iptest() |
|
323 | 343 | |
|
324 | 344 | |
|
325 | 345 | if __name__ == '__main__': |
|
326 | 346 | main() |
@@ -1,296 +1,295 b'' | |||
|
1 | 1 | """Tests for various magic functions. |
|
2 | 2 | |
|
3 | 3 | Needs to be run by nose (to make ipython session available). |
|
4 | 4 | """ |
|
5 | 5 | |
|
6 | 6 | import os |
|
7 | 7 | import sys |
|
8 | 8 | import tempfile |
|
9 | 9 | import types |
|
10 | 10 | |
|
11 | 11 | import nose.tools as nt |
|
12 | 12 | |
|
13 | 13 | from IPython.platutils import find_cmd, get_long_path_name |
|
14 | 14 | from IPython.testing import decorators as dec |
|
15 | 15 | from IPython.testing import tools as tt |
|
16 | 16 | |
|
17 | 17 | #----------------------------------------------------------------------------- |
|
18 | 18 | # Test functions begin |
|
19 | 19 | |
|
20 | 20 | def test_rehashx(): |
|
21 | 21 | # clear up everything |
|
22 | 22 | _ip.IP.alias_table.clear() |
|
23 | 23 | del _ip.db['syscmdlist'] |
|
24 | 24 | |
|
25 | 25 | _ip.magic('rehashx') |
|
26 | 26 | # Practically ALL ipython development systems will have more than 10 aliases |
|
27 | 27 | |
|
28 | 28 | yield (nt.assert_true, len(_ip.IP.alias_table) > 10) |
|
29 | 29 | for key, val in _ip.IP.alias_table.items(): |
|
30 | 30 | # we must strip dots from alias names |
|
31 | 31 | nt.assert_true('.' not in key) |
|
32 | 32 | |
|
33 | 33 | # rehashx must fill up syscmdlist |
|
34 | 34 | scoms = _ip.db['syscmdlist'] |
|
35 | 35 | yield (nt.assert_true, len(scoms) > 10) |
|
36 | 36 | |
|
37 | 37 | |
|
38 | 38 | def doctest_hist_f(): |
|
39 | 39 | """Test %hist -f with temporary filename. |
|
40 | 40 | |
|
41 | 41 | In [9]: import tempfile |
|
42 | 42 | |
|
43 | 43 | In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-') |
|
44 | 44 | |
|
45 | 45 | In [11]: %hist -n -f $tfile 3 |
|
46 | ||
|
47 | 46 | """ |
|
48 | 47 | |
|
49 | 48 | |
|
50 | 49 | def doctest_hist_r(): |
|
51 | 50 | """Test %hist -r |
|
52 | 51 | |
|
53 | 52 | XXX - This test is not recording the output correctly. Not sure why... |
|
54 | 53 | |
|
55 | 54 | In [20]: 'hist' in _ip.IP.lsmagic() |
|
56 | 55 | Out[20]: True |
|
57 | 56 | |
|
58 | 57 | In [6]: x=1 |
|
59 | 58 | |
|
60 | 59 | In [7]: %hist -n -r 2 |
|
61 | 60 | x=1 # random |
|
62 | 61 | hist -n -r 2 # random |
|
63 | 62 | """ |
|
64 | 63 | |
|
65 | 64 | # This test is known to fail on win32. |
|
66 | 65 | # See ticket https://bugs.launchpad.net/bugs/366334 |
|
67 | 66 | def test_obj_del(): |
|
68 | 67 | """Test that object's __del__ methods are called on exit.""" |
|
69 | 68 | test_dir = os.path.dirname(__file__) |
|
70 | 69 | del_file = os.path.join(test_dir,'obj_del.py') |
|
71 | 70 | ipython_cmd = find_cmd('ipython') |
|
72 | 71 | out = _ip.IP.getoutput('%s %s' % (ipython_cmd, del_file)) |
|
73 | 72 | nt.assert_equals(out,'obj_del.py: object A deleted') |
|
74 | 73 | |
|
75 | 74 | |
|
76 | 75 | def test_shist(): |
|
77 | 76 | # Simple tests of ShadowHist class - test generator. |
|
78 | 77 | import os, shutil, tempfile |
|
79 | 78 | |
|
80 | 79 | from IPython.Extensions import pickleshare |
|
81 | 80 | from IPython.history import ShadowHist |
|
82 | 81 | |
|
83 | 82 | tfile = tempfile.mktemp('','tmp-ipython-') |
|
84 | 83 | |
|
85 | 84 | db = pickleshare.PickleShareDB(tfile) |
|
86 | 85 | s = ShadowHist(db) |
|
87 | 86 | s.add('hello') |
|
88 | 87 | s.add('world') |
|
89 | 88 | s.add('hello') |
|
90 | 89 | s.add('hello') |
|
91 | 90 | s.add('karhu') |
|
92 | 91 | |
|
93 | 92 | yield nt.assert_equals,s.all(),[(1, 'hello'), (2, 'world'), (3, 'karhu')] |
|
94 | 93 | |
|
95 | 94 | yield nt.assert_equal,s.get(2),'world' |
|
96 | 95 | |
|
97 | 96 | shutil.rmtree(tfile) |
|
98 | 97 | |
|
99 | 98 | @dec.skipif_not_numpy |
|
100 | 99 | def test_numpy_clear_array_undec(): |
|
101 | 100 | from IPython.Extensions import clearcmd |
|
102 | 101 | |
|
103 | 102 | _ip.ex('import numpy as np') |
|
104 | 103 | _ip.ex('a = np.empty(2)') |
|
105 | 104 | yield (nt.assert_true, 'a' in _ip.user_ns) |
|
106 | 105 | _ip.magic('clear array') |
|
107 | 106 | yield (nt.assert_false, 'a' in _ip.user_ns) |
|
108 | 107 | |
|
109 | 108 | |
|
110 | 109 | @dec.skip() |
|
111 | 110 | def test_fail_dec(*a,**k): |
|
112 | 111 | yield nt.assert_true, False |
|
113 | 112 | |
|
114 | 113 | @dec.skip('This one shouldn not run') |
|
115 | 114 | def test_fail_dec2(*a,**k): |
|
116 | 115 | yield nt.assert_true, False |
|
117 | 116 | |
|
118 | 117 | @dec.skipknownfailure |
|
119 | 118 | def test_fail_dec3(*a,**k): |
|
120 | 119 | yield nt.assert_true, False |
|
121 | 120 | |
|
122 | 121 | |
|
123 | 122 | def doctest_refbug(): |
|
124 | 123 | """Very nasty problem with references held by multiple runs of a script. |
|
125 | 124 | See: https://bugs.launchpad.net/ipython/+bug/269966 |
|
126 | 125 | |
|
127 | 126 | In [1]: _ip.IP.clear_main_mod_cache() |
|
128 | 127 | |
|
129 | 128 | In [2]: run refbug |
|
130 | 129 | |
|
131 | 130 | In [3]: call_f() |
|
132 | 131 | lowercased: hello |
|
133 | 132 | |
|
134 | 133 | In [4]: run refbug |
|
135 | 134 | |
|
136 | 135 | In [5]: call_f() |
|
137 | 136 | lowercased: hello |
|
138 | 137 | lowercased: hello |
|
139 | 138 | """ |
|
140 | 139 | |
|
141 | 140 | #----------------------------------------------------------------------------- |
|
142 | 141 | # Tests for %run |
|
143 | 142 | #----------------------------------------------------------------------------- |
|
144 | 143 | |
|
145 | 144 | # %run is critical enough that it's a good idea to have a solid collection of |
|
146 | 145 | # tests for it, some as doctests and some as normal tests. |
|
147 | 146 | |
|
148 | 147 | def doctest_run_ns(): |
|
149 | 148 | """Classes declared %run scripts must be instantiable afterwards. |
|
150 | 149 | |
|
151 | 150 | In [11]: run tclass foo |
|
152 | 151 | |
|
153 | 152 | In [12]: isinstance(f(),foo) |
|
154 | 153 | Out[12]: True |
|
155 | 154 | """ |
|
156 | 155 | |
|
157 | 156 | |
|
158 | 157 | def doctest_run_ns2(): |
|
159 | 158 | """Classes declared %run scripts must be instantiable afterwards. |
|
160 | 159 | |
|
161 | 160 | In [4]: run tclass C-first_pass |
|
162 | 161 | |
|
163 | 162 | In [5]: run tclass C-second_pass |
|
164 | 163 | tclass.py: deleting object: C-first_pass |
|
165 | 164 | """ |
|
166 | 165 | |
|
167 | 166 | def doctest_run_builtins(): |
|
168 | 167 | """Check that %run doesn't damage __builtins__ via a doctest. |
|
169 | 168 | |
|
170 | 169 | This is similar to the test_run_builtins, but I want *both* forms of the |
|
171 | 170 | test to catch any possible glitches in our testing machinery, since that |
|
172 | 171 | modifies %run somewhat. So for this, we have both a normal test (below) |
|
173 | 172 | and a doctest (this one). |
|
174 | 173 | |
|
175 | 174 | In [1]: import tempfile |
|
176 | 175 | |
|
177 | 176 | In [2]: bid1 = id(__builtins__) |
|
178 | 177 | |
|
179 | 178 | In [3]: fname = tempfile.mkstemp()[1] |
|
180 | 179 | |
|
181 | 180 | In [3]: f = open(fname,'w') |
|
182 | 181 | |
|
183 | 182 | In [4]: f.write('pass\\n') |
|
184 | 183 | |
|
185 | 184 | In [5]: f.flush() |
|
186 | 185 | |
|
187 | 186 | In [6]: print type(__builtins__) |
|
188 | 187 | <type 'module'> |
|
189 | 188 | |
|
190 | 189 | In [7]: %run "$fname" |
|
191 | 190 | |
|
192 | 191 | In [7]: f.close() |
|
193 | 192 | |
|
194 | 193 | In [8]: bid2 = id(__builtins__) |
|
195 | 194 | |
|
196 | 195 | In [9]: print type(__builtins__) |
|
197 | 196 | <type 'module'> |
|
198 | 197 | |
|
199 | 198 | In [10]: bid1 == bid2 |
|
200 | 199 | Out[10]: True |
|
201 | 200 | |
|
202 | 201 | In [12]: try: |
|
203 | 202 | ....: os.unlink(fname) |
|
204 | 203 | ....: except: |
|
205 | 204 | ....: pass |
|
206 | 205 | ....: |
|
207 | 206 | """ |
|
208 | 207 | |
|
209 | 208 | # For some tests, it will be handy to organize them in a class with a common |
|
210 | 209 | # setup that makes a temp file |
|
211 | 210 | |
|
212 | 211 | class TestMagicRun(object): |
|
213 | 212 | |
|
214 | 213 | def setup(self): |
|
215 | 214 | """Make a valid python temp file.""" |
|
216 | 215 | fname = tempfile.mkstemp()[1] |
|
217 | 216 | f = open(fname,'w') |
|
218 | 217 | f.write('pass\n') |
|
219 | 218 | f.flush() |
|
220 | 219 | self.tmpfile = f |
|
221 | 220 | self.fname = fname |
|
222 | 221 | |
|
223 | 222 | def run_tmpfile(self): |
|
224 | 223 | # This fails on Windows if self.tmpfile.name has spaces or "~" in it. |
|
225 | 224 | # See below and ticket https://bugs.launchpad.net/bugs/366353 |
|
226 | 225 | _ip.magic('run "%s"' % self.fname) |
|
227 | 226 | |
|
228 | 227 | def test_builtins_id(self): |
|
229 | 228 | """Check that %run doesn't damage __builtins__ """ |
|
230 | 229 | |
|
231 | 230 | # Test that the id of __builtins__ is not modified by %run |
|
232 | 231 | bid1 = id(_ip.user_ns['__builtins__']) |
|
233 | 232 | self.run_tmpfile() |
|
234 | 233 | bid2 = id(_ip.user_ns['__builtins__']) |
|
235 | 234 | tt.assert_equals(bid1, bid2) |
|
236 | 235 | |
|
237 | 236 | def test_builtins_type(self): |
|
238 | 237 | """Check that the type of __builtins__ doesn't change with %run. |
|
239 | 238 | |
|
240 | 239 | However, the above could pass if __builtins__ was already modified to |
|
241 | 240 | be a dict (it should be a module) by a previous use of %run. So we |
|
242 | 241 | also check explicitly that it really is a module: |
|
243 | 242 | """ |
|
244 | 243 | self.run_tmpfile() |
|
245 | 244 | tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys)) |
|
246 | 245 | |
|
247 | 246 | def test_prompts(self): |
|
248 | 247 | """Test that prompts correctly generate after %run""" |
|
249 | 248 | self.run_tmpfile() |
|
250 | 249 | p2 = str(_ip.IP.outputcache.prompt2).strip() |
|
251 | 250 | nt.assert_equals(p2[:3], '...') |
|
252 | 251 | |
|
253 | 252 | def teardown(self): |
|
254 | 253 | self.tmpfile.close() |
|
255 | 254 | try: |
|
256 | 255 | os.unlink(self.fname) |
|
257 | 256 | except: |
|
258 | 257 | # On Windows, even though we close the file, we still can't delete |
|
259 | 258 | # it. I have no clue why |
|
260 | 259 | pass |
|
261 | 260 | |
|
262 | 261 | # Multiple tests for clipboard pasting |
|
263 | 262 | def test_paste(): |
|
264 | 263 | |
|
265 | 264 | def paste(txt): |
|
266 | 265 | hooks.clipboard_get = lambda : txt |
|
267 | 266 | _ip.magic('paste') |
|
268 | 267 | |
|
269 | 268 | # Inject fake clipboard hook but save original so we can restore it later |
|
270 | 269 | hooks = _ip.IP.hooks |
|
271 | 270 | user_ns = _ip.user_ns |
|
272 | 271 | original_clip = hooks.clipboard_get |
|
273 | 272 | |
|
274 | 273 | try: |
|
275 | 274 | # Run tests with fake clipboard function |
|
276 | 275 | user_ns.pop('x', None) |
|
277 | 276 | paste('x=1') |
|
278 | 277 | yield (nt.assert_equal, user_ns['x'], 1) |
|
279 | 278 | |
|
280 | 279 | user_ns.pop('x', None) |
|
281 | 280 | paste('>>> x=2') |
|
282 | 281 | yield (nt.assert_equal, user_ns['x'], 2) |
|
283 | 282 | |
|
284 | 283 | paste(""" |
|
285 | 284 | >>> x = [1,2,3] |
|
286 | 285 | >>> y = [] |
|
287 | 286 | >>> for i in x: |
|
288 | 287 | ... y.append(i**2) |
|
289 | 288 | ... |
|
290 | 289 | """) |
|
291 | 290 | yield (nt.assert_equal, user_ns['x'], [1,2,3]) |
|
292 | 291 | yield (nt.assert_equal, user_ns['y'], [1,4,9]) |
|
293 | 292 | |
|
294 | 293 | finally: |
|
295 | 294 | # Restore original hook |
|
296 | 295 | hooks.clipboard_get = original_clip |
@@ -1,282 +1,285 b'' | |||
|
1 | 1 | """Twisted shell support. |
|
2 | 2 | |
|
3 | 3 | XXX - This module is missing proper docs. |
|
4 | 4 | """ |
|
5 | # Tell nose to skip this module | |
|
6 | __test__ = {} | |
|
7 | ||
|
5 | 8 | import sys |
|
6 | 9 | |
|
7 | 10 | from twisted.internet import reactor, threads |
|
8 | 11 | |
|
9 | 12 | from IPython.ipmaker import make_IPython |
|
10 | 13 | from IPython.iplib import InteractiveShell |
|
11 | 14 | from IPython.ipstruct import Struct |
|
12 | 15 | import Queue,thread,threading,signal |
|
13 | 16 | from signal import signal, SIGINT |
|
14 | 17 | from IPython.genutils import Term,warn,error,flag_calls, ask_yes_no |
|
15 | 18 | import shellglobals |
|
16 | 19 | |
|
17 | 20 | def install_gtk2(): |
|
18 | 21 | """ Install gtk2 reactor, needs to be called bef """ |
|
19 | 22 | from twisted.internet import gtk2reactor |
|
20 | 23 | gtk2reactor.install() |
|
21 | 24 | |
|
22 | 25 | |
|
23 | 26 | def hijack_reactor(): |
|
24 | 27 | """Modifies Twisted's reactor with a dummy so user code does |
|
25 | 28 | not block IPython. This function returns the original |
|
26 | 29 | 'twisted.internet.reactor' that has been hijacked. |
|
27 | 30 | |
|
28 | 31 | NOTE: Make sure you call this *AFTER* you've installed |
|
29 | 32 | the reactor of your choice. |
|
30 | 33 | """ |
|
31 | 34 | from twisted import internet |
|
32 | 35 | orig_reactor = internet.reactor |
|
33 | 36 | |
|
34 | 37 | class DummyReactor(object): |
|
35 | 38 | def run(self): |
|
36 | 39 | pass |
|
37 | 40 | def __getattr__(self, name): |
|
38 | 41 | return getattr(orig_reactor, name) |
|
39 | 42 | def __setattr__(self, name, value): |
|
40 | 43 | return setattr(orig_reactor, name, value) |
|
41 | 44 | |
|
42 | 45 | internet.reactor = DummyReactor() |
|
43 | 46 | return orig_reactor |
|
44 | 47 | |
|
45 | 48 | class TwistedInteractiveShell(InteractiveShell): |
|
46 | 49 | """Simple multi-threaded shell.""" |
|
47 | 50 | |
|
48 | 51 | # Threading strategy taken from: |
|
49 | 52 | # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65109, by Brian |
|
50 | 53 | # McErlean and John Finlay. Modified with corrections by Antoon Pardon, |
|
51 | 54 | # from the pygtk mailing list, to avoid lockups with system calls. |
|
52 | 55 | |
|
53 | 56 | # class attribute to indicate whether the class supports threads or not. |
|
54 | 57 | # Subclasses with thread support should override this as needed. |
|
55 | 58 | isthreaded = True |
|
56 | 59 | |
|
57 | 60 | def __init__(self,name,usage=None,rc=Struct(opts=None,args=None), |
|
58 | 61 | user_ns=None,user_global_ns=None,banner2='',**kw): |
|
59 | 62 | """Similar to the normal InteractiveShell, but with threading control""" |
|
60 | 63 | |
|
61 | 64 | InteractiveShell.__init__(self,name,usage,rc,user_ns, |
|
62 | 65 | user_global_ns,banner2) |
|
63 | 66 | |
|
64 | 67 | |
|
65 | 68 | # A queue to hold the code to be executed. |
|
66 | 69 | self.code_queue = Queue.Queue() |
|
67 | 70 | |
|
68 | 71 | # Stuff to do at closing time |
|
69 | 72 | self._kill = None |
|
70 | 73 | on_kill = kw.get('on_kill', []) |
|
71 | 74 | # Check that all things to kill are callable: |
|
72 | 75 | for t in on_kill: |
|
73 | 76 | if not callable(t): |
|
74 | 77 | raise TypeError,'on_kill must be a list of callables' |
|
75 | 78 | self.on_kill = on_kill |
|
76 | 79 | # thread identity of the "worker thread" (that may execute code directly) |
|
77 | 80 | self.worker_ident = None |
|
78 | 81 | self.reactor_started = False |
|
79 | 82 | self.first_run = True |
|
80 | 83 | |
|
81 | 84 | def runsource(self, source, filename="<input>", symbol="single"): |
|
82 | 85 | """Compile and run some source in the interpreter. |
|
83 | 86 | |
|
84 | 87 | Modified version of code.py's runsource(), to handle threading issues. |
|
85 | 88 | See the original for full docstring details.""" |
|
86 | 89 | |
|
87 | 90 | # If Ctrl-C was typed, we reset the flag and return right away |
|
88 | 91 | if shellglobals.KBINT: |
|
89 | 92 | shellglobals.KBINT = False |
|
90 | 93 | return False |
|
91 | 94 | |
|
92 | 95 | if self._kill: |
|
93 | 96 | # can't queue new code if we are being killed |
|
94 | 97 | return True |
|
95 | 98 | |
|
96 | 99 | try: |
|
97 | 100 | code = self.compile(source, filename, symbol) |
|
98 | 101 | except (OverflowError, SyntaxError, ValueError): |
|
99 | 102 | # Case 1 |
|
100 | 103 | self.showsyntaxerror(filename) |
|
101 | 104 | return False |
|
102 | 105 | |
|
103 | 106 | if code is None: |
|
104 | 107 | # Case 2 |
|
105 | 108 | return True |
|
106 | 109 | |
|
107 | 110 | # shortcut - if we are in worker thread, or the worker thread is not running, |
|
108 | 111 | # execute directly (to allow recursion and prevent deadlock if code is run early |
|
109 | 112 | # in IPython construction) |
|
110 | 113 | |
|
111 | 114 | if (not self.reactor_started or (self.worker_ident is None and not self.first_run) |
|
112 | 115 | or self.worker_ident == thread.get_ident() or shellglobals.run_in_frontend(source)): |
|
113 | 116 | InteractiveShell.runcode(self,code) |
|
114 | 117 | return |
|
115 | 118 | |
|
116 | 119 | # Case 3 |
|
117 | 120 | # Store code in queue, so the execution thread can handle it. |
|
118 | 121 | |
|
119 | 122 | self.first_run = False |
|
120 | 123 | completed_ev, received_ev = threading.Event(), threading.Event() |
|
121 | 124 | |
|
122 | 125 | self.code_queue.put((code,completed_ev, received_ev)) |
|
123 | 126 | |
|
124 | 127 | reactor.callLater(0.0,self.runcode) |
|
125 | 128 | received_ev.wait(5) |
|
126 | 129 | if not received_ev.isSet(): |
|
127 | 130 | # the mainloop is dead, start executing code directly |
|
128 | 131 | print "Warning: Timeout for mainloop thread exceeded" |
|
129 | 132 | print "switching to nonthreaded mode (until mainloop wakes up again)" |
|
130 | 133 | self.worker_ident = None |
|
131 | 134 | else: |
|
132 | 135 | completed_ev.wait() |
|
133 | 136 | |
|
134 | 137 | return False |
|
135 | 138 | |
|
136 | 139 | def runcode(self): |
|
137 | 140 | """Execute a code object. |
|
138 | 141 | |
|
139 | 142 | Multithreaded wrapper around IPython's runcode().""" |
|
140 | 143 | |
|
141 | 144 | |
|
142 | 145 | # we are in worker thread, stash out the id for runsource() |
|
143 | 146 | self.worker_ident = thread.get_ident() |
|
144 | 147 | |
|
145 | 148 | if self._kill: |
|
146 | 149 | print >>Term.cout, 'Closing threads...', |
|
147 | 150 | Term.cout.flush() |
|
148 | 151 | for tokill in self.on_kill: |
|
149 | 152 | tokill() |
|
150 | 153 | print >>Term.cout, 'Done.' |
|
151 | 154 | # allow kill() to return |
|
152 | 155 | self._kill.set() |
|
153 | 156 | return True |
|
154 | 157 | |
|
155 | 158 | # Install SIGINT handler. We do it every time to ensure that if user |
|
156 | 159 | # code modifies it, we restore our own handling. |
|
157 | 160 | try: |
|
158 | 161 | pass |
|
159 | 162 | signal(SIGINT,shellglobals.sigint_handler) |
|
160 | 163 | except SystemError: |
|
161 | 164 | # This happens under Windows, which seems to have all sorts |
|
162 | 165 | # of problems with signal handling. Oh well... |
|
163 | 166 | pass |
|
164 | 167 | |
|
165 | 168 | # Flush queue of pending code by calling the run methood of the parent |
|
166 | 169 | # class with all items which may be in the queue. |
|
167 | 170 | code_to_run = None |
|
168 | 171 | while 1: |
|
169 | 172 | try: |
|
170 | 173 | code_to_run, completed_ev, received_ev = self.code_queue.get_nowait() |
|
171 | 174 | except Queue.Empty: |
|
172 | 175 | break |
|
173 | 176 | received_ev.set() |
|
174 | 177 | |
|
175 | 178 | |
|
176 | 179 | # Exceptions need to be raised differently depending on which |
|
177 | 180 | # thread is active. This convoluted try/except is only there to |
|
178 | 181 | # protect against asynchronous exceptions, to ensure that a shellglobals.KBINT |
|
179 | 182 | # at the wrong time doesn't deadlock everything. The global |
|
180 | 183 | # CODE_TO_RUN is set to true/false as close as possible to the |
|
181 | 184 | # runcode() call, so that the KBINT handler is correctly informed. |
|
182 | 185 | try: |
|
183 | 186 | try: |
|
184 | 187 | shellglobals.CODE_RUN = True |
|
185 | 188 | InteractiveShell.runcode(self,code_to_run) |
|
186 | 189 | except KeyboardInterrupt: |
|
187 | 190 | print "Keyboard interrupted in mainloop" |
|
188 | 191 | while not self.code_queue.empty(): |
|
189 | 192 | code = self.code_queue.get_nowait() |
|
190 | 193 | break |
|
191 | 194 | finally: |
|
192 | 195 | shellglobals.CODE_RUN = False |
|
193 | 196 | # allow runsource() return from wait |
|
194 | 197 | completed_ev.set() |
|
195 | 198 | |
|
196 | 199 | # This MUST return true for gtk threading to work |
|
197 | 200 | return True |
|
198 | 201 | |
|
199 | 202 | def kill(self): |
|
200 | 203 | """Kill the thread, returning when it has been shut down.""" |
|
201 | 204 | self._kill = threading.Event() |
|
202 | 205 | reactor.callLater(0.0,self.runcode) |
|
203 | 206 | self._kill.wait() |
|
204 | 207 | |
|
205 | 208 | |
|
206 | 209 | |
|
207 | 210 | class IPShellTwisted: |
|
208 | 211 | """Run a Twisted reactor while in an IPython session. |
|
209 | 212 | |
|
210 | 213 | Python commands can be passed to the thread where they will be |
|
211 | 214 | executed. This is implemented by periodically checking for |
|
212 | 215 | passed code using a Twisted reactor callback. |
|
213 | 216 | """ |
|
214 | 217 | |
|
215 | 218 | TIMEOUT = 0.01 # Millisecond interval between reactor runs. |
|
216 | 219 | |
|
217 | 220 | def __init__(self, argv=None, user_ns=None, debug=1, |
|
218 | 221 | shell_class=TwistedInteractiveShell): |
|
219 | 222 | |
|
220 | 223 | from twisted.internet import reactor |
|
221 | 224 | self.reactor = hijack_reactor() |
|
222 | 225 | |
|
223 | 226 | mainquit = self.reactor.stop |
|
224 | 227 | |
|
225 | 228 | # Make sure IPython keeps going after reactor stop. |
|
226 | 229 | def reactorstop(): |
|
227 | 230 | pass |
|
228 | 231 | self.reactor.stop = reactorstop |
|
229 | 232 | reactorrun_orig = self.reactor.run |
|
230 | 233 | self.quitting = False |
|
231 | 234 | def reactorrun(): |
|
232 | 235 | while True and not self.quitting: |
|
233 | 236 | reactorrun_orig() |
|
234 | 237 | self.reactor.run = reactorrun |
|
235 | 238 | |
|
236 | 239 | self.IP = make_IPython(argv, user_ns=user_ns, debug=debug, |
|
237 | 240 | shell_class=shell_class, |
|
238 | 241 | on_kill=[mainquit]) |
|
239 | 242 | |
|
240 | 243 | # threading.Thread.__init__(self) |
|
241 | 244 | |
|
242 | 245 | def run(self): |
|
243 | 246 | self.IP.mainloop() |
|
244 | 247 | self.quitting = True |
|
245 | 248 | self.IP.kill() |
|
246 | 249 | |
|
247 | 250 | def mainloop(self): |
|
248 | 251 | def mainLoopThreadDeath(r): |
|
249 | 252 | print "mainLoopThreadDeath: ", str(r) |
|
250 | 253 | def spawnMainloopThread(): |
|
251 | 254 | d=threads.deferToThread(self.run) |
|
252 | 255 | d.addBoth(mainLoopThreadDeath) |
|
253 | 256 | reactor.callWhenRunning(spawnMainloopThread) |
|
254 | 257 | self.IP.reactor_started = True |
|
255 | 258 | self.reactor.run() |
|
256 | 259 | print "mainloop ending...." |
|
257 | 260 | |
|
258 | 261 | exists = True |
|
259 | 262 | |
|
260 | 263 | |
|
261 | 264 | if __name__ == '__main__': |
|
262 | 265 | # Sample usage. |
|
263 | 266 | |
|
264 | 267 | # Create the shell object. This steals twisted.internet.reactor |
|
265 | 268 | # for its own purposes, to make sure you've already installed a |
|
266 | 269 | # reactor of your choice. |
|
267 | 270 | shell = IPShellTwisted( |
|
268 | 271 | argv=[], |
|
269 | 272 | user_ns={'__name__': '__example__', |
|
270 | 273 | 'hello': 'world', |
|
271 | 274 | }, |
|
272 | 275 | ) |
|
273 | 276 | |
|
274 | 277 | # Run the mainloop. This runs the actual reactor.run() method. |
|
275 | 278 | # The twisted.internet.reactor object at this point is a dummy |
|
276 | 279 | # object that passes through to the actual reactor, but prevents |
|
277 | 280 | # run() from being called on it again. |
|
278 | 281 | shell.mainloop() |
|
279 | 282 | |
|
280 | 283 | # You must exit IPython to terminate your program. |
|
281 | 284 | print 'Goodbye!' |
|
282 | 285 |
General Comments 0
You need to be logged in to leave comments.
Login now