Show More
@@ -1,296 +1,295 b'' | |||
|
1 | 1 | """Tests for various magic functions. |
|
2 | 2 | |
|
3 | 3 | Needs to be run by nose (to make ipython session available). |
|
4 | 4 | """ |
|
5 | 5 | |
|
6 | 6 | import os |
|
7 | 7 | import sys |
|
8 | 8 | import tempfile |
|
9 | 9 | import types |
|
10 | 10 | |
|
11 | 11 | import nose.tools as nt |
|
12 | 12 | |
|
13 | 13 | from IPython.utils.platutils import find_cmd, get_long_path_name |
|
14 | 14 | from IPython.testing import decorators as dec |
|
15 | 15 | from IPython.testing import tools as tt |
|
16 | 16 | |
|
17 | 17 | #----------------------------------------------------------------------------- |
|
18 | 18 | # Test functions begin |
|
19 | 19 | |
|
20 | 20 | def test_rehashx(): |
|
21 | 21 | # clear up everything |
|
22 | 22 | _ip.IP.alias_table.clear() |
|
23 | 23 | del _ip.db['syscmdlist'] |
|
24 | 24 | |
|
25 | 25 | _ip.magic('rehashx') |
|
26 | 26 | # Practically ALL ipython development systems will have more than 10 aliases |
|
27 | 27 | |
|
28 | 28 | yield (nt.assert_true, len(_ip.IP.alias_table) > 10) |
|
29 | 29 | for key, val in _ip.IP.alias_table.items(): |
|
30 | 30 | # we must strip dots from alias names |
|
31 | 31 | nt.assert_true('.' not in key) |
|
32 | 32 | |
|
33 | 33 | # rehashx must fill up syscmdlist |
|
34 | 34 | scoms = _ip.db['syscmdlist'] |
|
35 | 35 | yield (nt.assert_true, len(scoms) > 10) |
|
36 | 36 | |
|
37 | 37 | |
|
38 | 38 | def doctest_hist_f(): |
|
39 | 39 | """Test %hist -f with temporary filename. |
|
40 | 40 | |
|
41 | 41 | In [9]: import tempfile |
|
42 | 42 | |
|
43 | 43 | In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-') |
|
44 | 44 | |
|
45 | 45 | In [11]: %hist -n -f $tfile 3 |
|
46 | ||
|
47 | 46 | """ |
|
48 | 47 | |
|
49 | 48 | |
|
50 | 49 | def doctest_hist_r(): |
|
51 | 50 | """Test %hist -r |
|
52 | 51 | |
|
53 | 52 | XXX - This test is not recording the output correctly. Not sure why... |
|
54 | 53 | |
|
55 | 54 | In [20]: 'hist' in _ip.IP.lsmagic() |
|
56 | 55 | Out[20]: True |
|
57 | 56 | |
|
58 | 57 | In [6]: x=1 |
|
59 | 58 | |
|
60 | 59 | In [7]: %hist -n -r 2 |
|
61 | 60 | x=1 # random |
|
62 | 61 | hist -n -r 2 # random |
|
63 | 62 | """ |
|
64 | 63 | |
|
65 | 64 | # This test is known to fail on win32. |
|
66 | 65 | # See ticket https://bugs.launchpad.net/bugs/366334 |
|
67 | 66 | def test_obj_del(): |
|
68 | 67 | """Test that object's __del__ methods are called on exit.""" |
|
69 | 68 | test_dir = os.path.dirname(__file__) |
|
70 | 69 | del_file = os.path.join(test_dir,'obj_del.py') |
|
71 | 70 | ipython_cmd = find_cmd('ipython') |
|
72 | 71 | out = _ip.IP.getoutput('%s %s' % (ipython_cmd, del_file)) |
|
73 | 72 | nt.assert_equals(out,'obj_del.py: object A deleted') |
|
74 | 73 | |
|
75 | 74 | |
|
76 | 75 | def test_shist(): |
|
77 | 76 | # Simple tests of ShadowHist class - test generator. |
|
78 | 77 | import os, shutil, tempfile |
|
79 | 78 | |
|
80 | 79 | from IPython.extensions import pickleshare |
|
81 | 80 | from IPython.core.history import ShadowHist |
|
82 | 81 | |
|
83 | 82 | tfile = tempfile.mktemp('','tmp-ipython-') |
|
84 | 83 | |
|
85 | 84 | db = pickleshare.PickleShareDB(tfile) |
|
86 | 85 | s = ShadowHist(db) |
|
87 | 86 | s.add('hello') |
|
88 | 87 | s.add('world') |
|
89 | 88 | s.add('hello') |
|
90 | 89 | s.add('hello') |
|
91 | 90 | s.add('karhu') |
|
92 | 91 | |
|
93 | 92 | yield nt.assert_equals,s.all(),[(1, 'hello'), (2, 'world'), (3, 'karhu')] |
|
94 | 93 | |
|
95 | 94 | yield nt.assert_equal,s.get(2),'world' |
|
96 | 95 | |
|
97 | 96 | shutil.rmtree(tfile) |
|
98 | 97 | |
|
99 | 98 | @dec.skipif_not_numpy |
|
100 | 99 | def test_numpy_clear_array_undec(): |
|
101 | 100 | from IPython.extensions import clearcmd |
|
102 | 101 | |
|
103 | 102 | _ip.ex('import numpy as np') |
|
104 | 103 | _ip.ex('a = np.empty(2)') |
|
105 | 104 | yield (nt.assert_true, 'a' in _ip.user_ns) |
|
106 | 105 | _ip.magic('clear array') |
|
107 | 106 | yield (nt.assert_false, 'a' in _ip.user_ns) |
|
108 | 107 | |
|
109 | 108 | |
|
110 | 109 | @dec.skip() |
|
111 | 110 | def test_fail_dec(*a,**k): |
|
112 | 111 | yield nt.assert_true, False |
|
113 | 112 | |
|
114 | 113 | @dec.skip('This one shouldn not run') |
|
115 | 114 | def test_fail_dec2(*a,**k): |
|
116 | 115 | yield nt.assert_true, False |
|
117 | 116 | |
|
118 | 117 | @dec.skipknownfailure |
|
119 | 118 | def test_fail_dec3(*a,**k): |
|
120 | 119 | yield nt.assert_true, False |
|
121 | 120 | |
|
122 | 121 | |
|
123 | 122 | def doctest_refbug(): |
|
124 | 123 | """Very nasty problem with references held by multiple runs of a script. |
|
125 | 124 | See: https://bugs.launchpad.net/ipython/+bug/269966 |
|
126 | 125 | |
|
127 | 126 | In [1]: _ip.IP.clear_main_mod_cache() |
|
128 | 127 | |
|
129 | 128 | In [2]: run refbug |
|
130 | 129 | |
|
131 | 130 | In [3]: call_f() |
|
132 | 131 | lowercased: hello |
|
133 | 132 | |
|
134 | 133 | In [4]: run refbug |
|
135 | 134 | |
|
136 | 135 | In [5]: call_f() |
|
137 | 136 | lowercased: hello |
|
138 | 137 | lowercased: hello |
|
139 | 138 | """ |
|
140 | 139 | |
|
141 | 140 | #----------------------------------------------------------------------------- |
|
142 | 141 | # Tests for %run |
|
143 | 142 | #----------------------------------------------------------------------------- |
|
144 | 143 | |
|
145 | 144 | # %run is critical enough that it's a good idea to have a solid collection of |
|
146 | 145 | # tests for it, some as doctests and some as normal tests. |
|
147 | 146 | |
|
148 | 147 | def doctest_run_ns(): |
|
149 | 148 | """Classes declared %run scripts must be instantiable afterwards. |
|
150 | 149 | |
|
151 | 150 | In [11]: run tclass foo |
|
152 | 151 | |
|
153 | 152 | In [12]: isinstance(f(),foo) |
|
154 | 153 | Out[12]: True |
|
155 | 154 | """ |
|
156 | 155 | |
|
157 | 156 | |
|
158 | 157 | def doctest_run_ns2(): |
|
159 | 158 | """Classes declared %run scripts must be instantiable afterwards. |
|
160 | 159 | |
|
161 | 160 | In [4]: run tclass C-first_pass |
|
162 | 161 | |
|
163 | 162 | In [5]: run tclass C-second_pass |
|
164 | 163 | tclass.py: deleting object: C-first_pass |
|
165 | 164 | """ |
|
166 | 165 | |
|
167 | 166 | def doctest_run_builtins(): |
|
168 | 167 | """Check that %run doesn't damage __builtins__ via a doctest. |
|
169 | 168 | |
|
170 | 169 | This is similar to the test_run_builtins, but I want *both* forms of the |
|
171 | 170 | test to catch any possible glitches in our testing machinery, since that |
|
172 | 171 | modifies %run somewhat. So for this, we have both a normal test (below) |
|
173 | 172 | and a doctest (this one). |
|
174 | 173 | |
|
175 | 174 | In [1]: import tempfile |
|
176 | 175 | |
|
177 | 176 | In [2]: bid1 = id(__builtins__) |
|
178 | 177 | |
|
179 | 178 | In [3]: fname = tempfile.mkstemp()[1] |
|
180 | 179 | |
|
181 | 180 | In [3]: f = open(fname,'w') |
|
182 | 181 | |
|
183 | 182 | In [4]: f.write('pass\\n') |
|
184 | 183 | |
|
185 | 184 | In [5]: f.flush() |
|
186 | 185 | |
|
187 | 186 | In [6]: print type(__builtins__) |
|
188 | 187 | <type 'module'> |
|
189 | 188 | |
|
190 | 189 | In [7]: %run "$fname" |
|
191 | 190 | |
|
192 | 191 | In [7]: f.close() |
|
193 | 192 | |
|
194 | 193 | In [8]: bid2 = id(__builtins__) |
|
195 | 194 | |
|
196 | 195 | In [9]: print type(__builtins__) |
|
197 | 196 | <type 'module'> |
|
198 | 197 | |
|
199 | 198 | In [10]: bid1 == bid2 |
|
200 | 199 | Out[10]: True |
|
201 | 200 | |
|
202 | 201 | In [12]: try: |
|
203 | 202 | ....: os.unlink(fname) |
|
204 | 203 | ....: except: |
|
205 | 204 | ....: pass |
|
206 | 205 | ....: |
|
207 | 206 | """ |
|
208 | 207 | |
|
209 | 208 | # For some tests, it will be handy to organize them in a class with a common |
|
210 | 209 | # setup that makes a temp file |
|
211 | 210 | |
|
212 | 211 | class TestMagicRun(object): |
|
213 | 212 | |
|
214 | 213 | def setup(self): |
|
215 | 214 | """Make a valid python temp file.""" |
|
216 | 215 | fname = tempfile.mkstemp()[1] |
|
217 | 216 | f = open(fname,'w') |
|
218 | 217 | f.write('pass\n') |
|
219 | 218 | f.flush() |
|
220 | 219 | self.tmpfile = f |
|
221 | 220 | self.fname = fname |
|
222 | 221 | |
|
223 | 222 | def run_tmpfile(self): |
|
224 | 223 | # This fails on Windows if self.tmpfile.name has spaces or "~" in it. |
|
225 | 224 | # See below and ticket https://bugs.launchpad.net/bugs/366353 |
|
226 | 225 | _ip.magic('run "%s"' % self.fname) |
|
227 | 226 | |
|
228 | 227 | def test_builtins_id(self): |
|
229 | 228 | """Check that %run doesn't damage __builtins__ """ |
|
230 | 229 | |
|
231 | 230 | # Test that the id of __builtins__ is not modified by %run |
|
232 | 231 | bid1 = id(_ip.user_ns['__builtins__']) |
|
233 | 232 | self.run_tmpfile() |
|
234 | 233 | bid2 = id(_ip.user_ns['__builtins__']) |
|
235 | 234 | tt.assert_equals(bid1, bid2) |
|
236 | 235 | |
|
237 | 236 | def test_builtins_type(self): |
|
238 | 237 | """Check that the type of __builtins__ doesn't change with %run. |
|
239 | 238 | |
|
240 | 239 | However, the above could pass if __builtins__ was already modified to |
|
241 | 240 | be a dict (it should be a module) by a previous use of %run. So we |
|
242 | 241 | also check explicitly that it really is a module: |
|
243 | 242 | """ |
|
244 | 243 | self.run_tmpfile() |
|
245 | 244 | tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys)) |
|
246 | 245 | |
|
247 | 246 | def test_prompts(self): |
|
248 | 247 | """Test that prompts correctly generate after %run""" |
|
249 | 248 | self.run_tmpfile() |
|
250 | 249 | p2 = str(_ip.IP.outputcache.prompt2).strip() |
|
251 | 250 | nt.assert_equals(p2[:3], '...') |
|
252 | 251 | |
|
253 | 252 | def teardown(self): |
|
254 | 253 | self.tmpfile.close() |
|
255 | 254 | try: |
|
256 | 255 | os.unlink(self.fname) |
|
257 | 256 | except: |
|
258 | 257 | # On Windows, even though we close the file, we still can't delete |
|
259 | 258 | # it. I have no clue why |
|
260 | 259 | pass |
|
261 | 260 | |
|
262 | 261 | # Multiple tests for clipboard pasting |
|
263 | 262 | def test_paste(): |
|
264 | 263 | |
|
265 | 264 | def paste(txt): |
|
266 | 265 | hooks.clipboard_get = lambda : txt |
|
267 | 266 | _ip.magic('paste') |
|
268 | 267 | |
|
269 | 268 | # Inject fake clipboard hook but save original so we can restore it later |
|
270 | 269 | hooks = _ip.IP.hooks |
|
271 | 270 | user_ns = _ip.user_ns |
|
272 | 271 | original_clip = hooks.clipboard_get |
|
273 | 272 | |
|
274 | 273 | try: |
|
275 | 274 | # Run tests with fake clipboard function |
|
276 | 275 | user_ns.pop('x', None) |
|
277 | 276 | paste('x=1') |
|
278 | 277 | yield (nt.assert_equal, user_ns['x'], 1) |
|
279 | 278 | |
|
280 | 279 | user_ns.pop('x', None) |
|
281 | 280 | paste('>>> x=2') |
|
282 | 281 | yield (nt.assert_equal, user_ns['x'], 2) |
|
283 | 282 | |
|
284 | 283 | paste(""" |
|
285 | 284 | >>> x = [1,2,3] |
|
286 | 285 | >>> y = [] |
|
287 | 286 | >>> for i in x: |
|
288 | 287 | ... y.append(i**2) |
|
289 | 288 | ... |
|
290 | 289 | """) |
|
291 | 290 | yield (nt.assert_equal, user_ns['x'], [1,2,3]) |
|
292 | 291 | yield (nt.assert_equal, user_ns['y'], [1,4,9]) |
|
293 | 292 | |
|
294 | 293 | finally: |
|
295 | 294 | # Restore original hook |
|
296 | 295 | hooks.clipboard_get = original_clip |
@@ -1,282 +1,285 b'' | |||
|
1 | 1 | """Twisted shell support. |
|
2 | 2 | |
|
3 | 3 | XXX - This module is missing proper docs. |
|
4 | 4 | """ |
|
5 | # Tell nose to skip this module | |
|
6 | __test__ = {} | |
|
7 | ||
|
5 | 8 | import sys |
|
6 | 9 | |
|
7 | 10 | from twisted.internet import reactor, threads |
|
8 | 11 | |
|
9 | 12 | from IPython.core.ipmaker import make_IPython |
|
10 | 13 | from IPython.core.iplib import InteractiveShell |
|
11 | 14 | from IPython.utils.ipstruct import Struct |
|
12 | 15 | import Queue,thread,threading,signal |
|
13 | 16 | from signal import signal, SIGINT |
|
14 | 17 | from IPython.utils.genutils import Term,warn,error,flag_calls, ask_yes_no |
|
15 | 18 | from IPython.core import shellglobals |
|
16 | 19 | |
|
17 | 20 | def install_gtk2(): |
|
18 | 21 | """ Install gtk2 reactor, needs to be called bef """ |
|
19 | 22 | from twisted.internet import gtk2reactor |
|
20 | 23 | gtk2reactor.install() |
|
21 | 24 | |
|
22 | 25 | |
|
23 | 26 | def hijack_reactor(): |
|
24 | 27 | """Modifies Twisted's reactor with a dummy so user code does |
|
25 | 28 | not block IPython. This function returns the original |
|
26 | 29 | 'twisted.internet.reactor' that has been hijacked. |
|
27 | 30 | |
|
28 | 31 | NOTE: Make sure you call this *AFTER* you've installed |
|
29 | 32 | the reactor of your choice. |
|
30 | 33 | """ |
|
31 | 34 | from twisted import internet |
|
32 | 35 | orig_reactor = internet.reactor |
|
33 | 36 | |
|
34 | 37 | class DummyReactor(object): |
|
35 | 38 | def run(self): |
|
36 | 39 | pass |
|
37 | 40 | def __getattr__(self, name): |
|
38 | 41 | return getattr(orig_reactor, name) |
|
39 | 42 | def __setattr__(self, name, value): |
|
40 | 43 | return setattr(orig_reactor, name, value) |
|
41 | 44 | |
|
42 | 45 | internet.reactor = DummyReactor() |
|
43 | 46 | return orig_reactor |
|
44 | 47 | |
|
45 | 48 | class TwistedInteractiveShell(InteractiveShell): |
|
46 | 49 | """Simple multi-threaded shell.""" |
|
47 | 50 | |
|
48 | 51 | # Threading strategy taken from: |
|
49 | 52 | # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65109, by Brian |
|
50 | 53 | # McErlean and John Finlay. Modified with corrections by Antoon Pardon, |
|
51 | 54 | # from the pygtk mailing list, to avoid lockups with system calls. |
|
52 | 55 | |
|
53 | 56 | # class attribute to indicate whether the class supports threads or not. |
|
54 | 57 | # Subclasses with thread support should override this as needed. |
|
55 | 58 | isthreaded = True |
|
56 | 59 | |
|
57 | 60 | def __init__(self,name,usage=None,rc=Struct(opts=None,args=None), |
|
58 | 61 | user_ns=None,user_global_ns=None,banner2='',**kw): |
|
59 | 62 | """Similar to the normal InteractiveShell, but with threading control""" |
|
60 | 63 | |
|
61 | 64 | InteractiveShell.__init__(self,name,usage,rc,user_ns, |
|
62 | 65 | user_global_ns,banner2) |
|
63 | 66 | |
|
64 | 67 | |
|
65 | 68 | # A queue to hold the code to be executed. |
|
66 | 69 | self.code_queue = Queue.Queue() |
|
67 | 70 | |
|
68 | 71 | # Stuff to do at closing time |
|
69 | 72 | self._kill = None |
|
70 | 73 | on_kill = kw.get('on_kill', []) |
|
71 | 74 | # Check that all things to kill are callable: |
|
72 | 75 | for t in on_kill: |
|
73 | 76 | if not callable(t): |
|
74 | 77 | raise TypeError,'on_kill must be a list of callables' |
|
75 | 78 | self.on_kill = on_kill |
|
76 | 79 | # thread identity of the "worker thread" (that may execute code directly) |
|
77 | 80 | self.worker_ident = None |
|
78 | 81 | self.reactor_started = False |
|
79 | 82 | self.first_run = True |
|
80 | 83 | |
|
81 | 84 | def runsource(self, source, filename="<input>", symbol="single"): |
|
82 | 85 | """Compile and run some source in the interpreter. |
|
83 | 86 | |
|
84 | 87 | Modified version of code.py's runsource(), to handle threading issues. |
|
85 | 88 | See the original for full docstring details.""" |
|
86 | 89 | |
|
87 | 90 | # If Ctrl-C was typed, we reset the flag and return right away |
|
88 | 91 | if shellglobals.KBINT: |
|
89 | 92 | shellglobals.KBINT = False |
|
90 | 93 | return False |
|
91 | 94 | |
|
92 | 95 | if self._kill: |
|
93 | 96 | # can't queue new code if we are being killed |
|
94 | 97 | return True |
|
95 | 98 | |
|
96 | 99 | try: |
|
97 | 100 | code = self.compile(source, filename, symbol) |
|
98 | 101 | except (OverflowError, SyntaxError, ValueError): |
|
99 | 102 | # Case 1 |
|
100 | 103 | self.showsyntaxerror(filename) |
|
101 | 104 | return False |
|
102 | 105 | |
|
103 | 106 | if code is None: |
|
104 | 107 | # Case 2 |
|
105 | 108 | return True |
|
106 | 109 | |
|
107 | 110 | # shortcut - if we are in worker thread, or the worker thread is not running, |
|
108 | 111 | # execute directly (to allow recursion and prevent deadlock if code is run early |
|
109 | 112 | # in IPython construction) |
|
110 | 113 | |
|
111 | 114 | if (not self.reactor_started or (self.worker_ident is None and not self.first_run) |
|
112 | 115 | or self.worker_ident == thread.get_ident() or shellglobals.run_in_frontend(source)): |
|
113 | 116 | InteractiveShell.runcode(self,code) |
|
114 | 117 | return |
|
115 | 118 | |
|
116 | 119 | # Case 3 |
|
117 | 120 | # Store code in queue, so the execution thread can handle it. |
|
118 | 121 | |
|
119 | 122 | self.first_run = False |
|
120 | 123 | completed_ev, received_ev = threading.Event(), threading.Event() |
|
121 | 124 | |
|
122 | 125 | self.code_queue.put((code,completed_ev, received_ev)) |
|
123 | 126 | |
|
124 | 127 | reactor.callLater(0.0,self.runcode) |
|
125 | 128 | received_ev.wait(5) |
|
126 | 129 | if not received_ev.isSet(): |
|
127 | 130 | # the mainloop is dead, start executing code directly |
|
128 | 131 | print "Warning: Timeout for mainloop thread exceeded" |
|
129 | 132 | print "switching to nonthreaded mode (until mainloop wakes up again)" |
|
130 | 133 | self.worker_ident = None |
|
131 | 134 | else: |
|
132 | 135 | completed_ev.wait() |
|
133 | 136 | |
|
134 | 137 | return False |
|
135 | 138 | |
|
136 | 139 | def runcode(self): |
|
137 | 140 | """Execute a code object. |
|
138 | 141 | |
|
139 | 142 | Multithreaded wrapper around IPython's runcode().""" |
|
140 | 143 | |
|
141 | 144 | |
|
142 | 145 | # we are in worker thread, stash out the id for runsource() |
|
143 | 146 | self.worker_ident = thread.get_ident() |
|
144 | 147 | |
|
145 | 148 | if self._kill: |
|
146 | 149 | print >>Term.cout, 'Closing threads...', |
|
147 | 150 | Term.cout.flush() |
|
148 | 151 | for tokill in self.on_kill: |
|
149 | 152 | tokill() |
|
150 | 153 | print >>Term.cout, 'Done.' |
|
151 | 154 | # allow kill() to return |
|
152 | 155 | self._kill.set() |
|
153 | 156 | return True |
|
154 | 157 | |
|
155 | 158 | # Install SIGINT handler. We do it every time to ensure that if user |
|
156 | 159 | # code modifies it, we restore our own handling. |
|
157 | 160 | try: |
|
158 | 161 | pass |
|
159 | 162 | signal(SIGINT,shellglobals.sigint_handler) |
|
160 | 163 | except SystemError: |
|
161 | 164 | # This happens under Windows, which seems to have all sorts |
|
162 | 165 | # of problems with signal handling. Oh well... |
|
163 | 166 | pass |
|
164 | 167 | |
|
165 | 168 | # Flush queue of pending code by calling the run methood of the parent |
|
166 | 169 | # class with all items which may be in the queue. |
|
167 | 170 | code_to_run = None |
|
168 | 171 | while 1: |
|
169 | 172 | try: |
|
170 | 173 | code_to_run, completed_ev, received_ev = self.code_queue.get_nowait() |
|
171 | 174 | except Queue.Empty: |
|
172 | 175 | break |
|
173 | 176 | received_ev.set() |
|
174 | 177 | |
|
175 | 178 | |
|
176 | 179 | # Exceptions need to be raised differently depending on which |
|
177 | 180 | # thread is active. This convoluted try/except is only there to |
|
178 | 181 | # protect against asynchronous exceptions, to ensure that a shellglobals.KBINT |
|
179 | 182 | # at the wrong time doesn't deadlock everything. The global |
|
180 | 183 | # CODE_TO_RUN is set to true/false as close as possible to the |
|
181 | 184 | # runcode() call, so that the KBINT handler is correctly informed. |
|
182 | 185 | try: |
|
183 | 186 | try: |
|
184 | 187 | shellglobals.CODE_RUN = True |
|
185 | 188 | InteractiveShell.runcode(self,code_to_run) |
|
186 | 189 | except KeyboardInterrupt: |
|
187 | 190 | print "Keyboard interrupted in mainloop" |
|
188 | 191 | while not self.code_queue.empty(): |
|
189 | 192 | code = self.code_queue.get_nowait() |
|
190 | 193 | break |
|
191 | 194 | finally: |
|
192 | 195 | shellglobals.CODE_RUN = False |
|
193 | 196 | # allow runsource() return from wait |
|
194 | 197 | completed_ev.set() |
|
195 | 198 | |
|
196 | 199 | # This MUST return true for gtk threading to work |
|
197 | 200 | return True |
|
198 | 201 | |
|
199 | 202 | def kill(self): |
|
200 | 203 | """Kill the thread, returning when it has been shut down.""" |
|
201 | 204 | self._kill = threading.Event() |
|
202 | 205 | reactor.callLater(0.0,self.runcode) |
|
203 | 206 | self._kill.wait() |
|
204 | 207 | |
|
205 | 208 | |
|
206 | 209 | |
|
207 | 210 | class IPShellTwisted: |
|
208 | 211 | """Run a Twisted reactor while in an IPython session. |
|
209 | 212 | |
|
210 | 213 | Python commands can be passed to the thread where they will be |
|
211 | 214 | executed. This is implemented by periodically checking for |
|
212 | 215 | passed code using a Twisted reactor callback. |
|
213 | 216 | """ |
|
214 | 217 | |
|
215 | 218 | TIMEOUT = 0.01 # Millisecond interval between reactor runs. |
|
216 | 219 | |
|
217 | 220 | def __init__(self, argv=None, user_ns=None, debug=1, |
|
218 | 221 | shell_class=TwistedInteractiveShell): |
|
219 | 222 | |
|
220 | 223 | from twisted.internet import reactor |
|
221 | 224 | self.reactor = hijack_reactor() |
|
222 | 225 | |
|
223 | 226 | mainquit = self.reactor.stop |
|
224 | 227 | |
|
225 | 228 | # Make sure IPython keeps going after reactor stop. |
|
226 | 229 | def reactorstop(): |
|
227 | 230 | pass |
|
228 | 231 | self.reactor.stop = reactorstop |
|
229 | 232 | reactorrun_orig = self.reactor.run |
|
230 | 233 | self.quitting = False |
|
231 | 234 | def reactorrun(): |
|
232 | 235 | while True and not self.quitting: |
|
233 | 236 | reactorrun_orig() |
|
234 | 237 | self.reactor.run = reactorrun |
|
235 | 238 | |
|
236 | 239 | self.IP = make_IPython(argv, user_ns=user_ns, debug=debug, |
|
237 | 240 | shell_class=shell_class, |
|
238 | 241 | on_kill=[mainquit]) |
|
239 | 242 | |
|
240 | 243 | # threading.Thread.__init__(self) |
|
241 | 244 | |
|
242 | 245 | def run(self): |
|
243 | 246 | self.IP.mainloop() |
|
244 | 247 | self.quitting = True |
|
245 | 248 | self.IP.kill() |
|
246 | 249 | |
|
247 | 250 | def mainloop(self): |
|
248 | 251 | def mainLoopThreadDeath(r): |
|
249 | 252 | print "mainLoopThreadDeath: ", str(r) |
|
250 | 253 | def spawnMainloopThread(): |
|
251 | 254 | d=threads.deferToThread(self.run) |
|
252 | 255 | d.addBoth(mainLoopThreadDeath) |
|
253 | 256 | reactor.callWhenRunning(spawnMainloopThread) |
|
254 | 257 | self.IP.reactor_started = True |
|
255 | 258 | self.reactor.run() |
|
256 | 259 | print "mainloop ending...." |
|
257 | 260 | |
|
258 | 261 | exists = True |
|
259 | 262 | |
|
260 | 263 | |
|
261 | 264 | if __name__ == '__main__': |
|
262 | 265 | # Sample usage. |
|
263 | 266 | |
|
264 | 267 | # Create the shell object. This steals twisted.internet.reactor |
|
265 | 268 | # for its own purposes, to make sure you've already installed a |
|
266 | 269 | # reactor of your choice. |
|
267 | 270 | shell = IPShellTwisted( |
|
268 | 271 | argv=[], |
|
269 | 272 | user_ns={'__name__': '__example__', |
|
270 | 273 | 'hello': 'world', |
|
271 | 274 | }, |
|
272 | 275 | ) |
|
273 | 276 | |
|
274 | 277 | # Run the mainloop. This runs the actual reactor.run() method. |
|
275 | 278 | # The twisted.internet.reactor object at this point is a dummy |
|
276 | 279 | # object that passes through to the actual reactor, but prevents |
|
277 | 280 | # run() from being called on it again. |
|
278 | 281 | shell.mainloop() |
|
279 | 282 | |
|
280 | 283 | # You must exit IPython to terminate your program. |
|
281 | 284 | print 'Goodbye!' |
|
282 | 285 |
@@ -1,77 +1,82 b'' | |||
|
1 | 1 | """ |
|
2 | 2 | Base front end class for all async frontends. |
|
3 | 3 | """ |
|
4 | 4 | __docformat__ = "restructuredtext en" |
|
5 | 5 | |
|
6 | # Tell nose to skip this module | |
|
7 | __test__ = {} | |
|
8 | ||
|
6 | 9 | #------------------------------------------------------------------------------- |
|
7 | 10 | # Copyright (C) 2008 The IPython Development Team |
|
8 | 11 | # |
|
9 | 12 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | 13 | # the file COPYING, distributed as part of this software. |
|
11 | 14 | #------------------------------------------------------------------------------- |
|
12 | 15 | |
|
13 | ||
|
14 | 16 | #------------------------------------------------------------------------------- |
|
15 | 17 | # Imports |
|
16 | 18 | #------------------------------------------------------------------------------- |
|
17 | 19 | |
|
20 | # Third-party | |
|
21 | from twisted.python.failure import Failure | |
|
22 | from zope.interface import implements, classProvides | |
|
23 | ||
|
24 | # From IPython | |
|
18 | 25 | from IPython.external import guid |
|
19 | 26 | |
|
20 | from zope.interface import Interface, Attribute, implements, classProvides | |
|
21 | from twisted.python.failure import Failure | |
|
22 | from IPython.frontend.frontendbase import ( | |
|
23 | FrontEndBase, IFrontEnd, IFrontEndFactory) | |
|
27 | from IPython.frontend.frontendbase import (FrontEndBase, IFrontEnd, | |
|
28 | IFrontEndFactory) | |
|
24 | 29 | from IPython.kernel.core.history import FrontEndHistory |
|
25 | 30 | from IPython.kernel.engineservice import IEngineCore |
|
26 | 31 | |
|
32 | #----------------------------------------------------------------------------- | |
|
33 | # Classes and functions | |
|
34 | #----------------------------------------------------------------------------- | |
|
27 | 35 | |
|
28 | 36 | class AsyncFrontEndBase(FrontEndBase): |
|
29 | 37 | """ |
|
30 | 38 | Overrides FrontEndBase to wrap execute in a deferred result. |
|
31 | 39 | All callbacks are made as callbacks on the deferred result. |
|
32 | 40 | """ |
|
33 | 41 | |
|
34 | 42 | implements(IFrontEnd) |
|
35 | 43 | classProvides(IFrontEndFactory) |
|
36 | 44 | |
|
37 | 45 | def __init__(self, engine=None, history=None): |
|
38 | 46 | assert(engine==None or IEngineCore.providedBy(engine)) |
|
39 | 47 | self.engine = IEngineCore(engine) |
|
40 | 48 | if history is None: |
|
41 | 49 | self.history = FrontEndHistory(input_cache=['']) |
|
42 | 50 | else: |
|
43 | 51 | self.history = history |
|
44 | ||
|
45 | ||
|
52 | ||
|
46 | 53 | def execute(self, block, blockID=None): |
|
47 | 54 | """Execute the block and return the deferred result. |
|
48 | 55 | |
|
49 | 56 | Parameters: |
|
50 | 57 | block : {str, AST} |
|
51 | 58 | blockID : any |
|
52 | 59 | Caller may provide an ID to identify this block. |
|
53 | 60 | result['blockID'] := blockID |
|
54 | 61 | |
|
55 | 62 | Result: |
|
56 | 63 | Deferred result of self.interpreter.execute |
|
57 | 64 | """ |
|
58 | 65 | |
|
59 | 66 | if(not self.is_complete(block)): |
|
60 | 67 | return Failure(Exception("Block is not compilable")) |
|
61 | 68 | |
|
62 | 69 | if(blockID == None): |
|
63 | 70 | blockID = guid.generate() |
|
64 | 71 | |
|
65 | 72 | d = self.engine.execute(block) |
|
66 | 73 | d.addCallback(self._add_history, block=block) |
|
67 | 74 | d.addCallbacks(self._add_block_id_for_result, |
|
68 | 75 | errback=self._add_block_id_for_failure, |
|
69 | 76 | callbackArgs=(blockID,), |
|
70 | 77 | errbackArgs=(blockID,)) |
|
71 | 78 | d.addBoth(self.update_cell_prompt, blockID=blockID) |
|
72 | 79 | d.addCallbacks(self.render_result, |
|
73 | 80 | errback=self.render_error) |
|
74 | 81 | |
|
75 | 82 | return d |
|
76 | ||
|
77 |
@@ -1,109 +1,112 b'' | |||
|
1 | 1 | # encoding: utf-8 |
|
2 | 2 | |
|
3 | 3 | """This file contains unittests for the asyncfrontendbase module.""" |
|
4 | 4 | |
|
5 | 5 | __docformat__ = "restructuredtext en" |
|
6 | ||
|
7 | # Tell nose to skip this module | |
|
8 | __test__ = {} | |
|
6 | 9 | |
|
7 | 10 | #--------------------------------------------------------------------------- |
|
8 | 11 | # Copyright (C) 2008 The IPython Development Team |
|
9 | 12 | # |
|
10 | 13 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | 14 | # the file COPYING, distributed as part of this software. |
|
12 | 15 | #--------------------------------------------------------------------------- |
|
13 | ||
|
16 | ||
|
14 | 17 | #--------------------------------------------------------------------------- |
|
15 | 18 | # Imports |
|
16 | 19 | #--------------------------------------------------------------------------- |
|
17 | 20 | |
|
18 | # Tell nose to skip this module | |
|
19 | __test__ = {} | |
|
20 | ||
|
21 | 21 | from twisted.trial import unittest |
|
22 | ||
|
22 | 23 | from IPython.frontend.asyncfrontendbase import AsyncFrontEndBase |
|
23 | 24 | from IPython.frontend import frontendbase |
|
24 | 25 | from IPython.kernel.engineservice import EngineService |
|
25 | 26 | from IPython.testing.parametric import Parametric, parametric |
|
26 | 27 | |
|
28 | #----------------------------------------------------------------------------- | |
|
29 | # Classes and functions | |
|
30 | #----------------------------------------------------------------------------- | |
|
27 | 31 | |
|
28 | 32 | class FrontEndCallbackChecker(AsyncFrontEndBase): |
|
29 | 33 | """FrontEndBase subclass for checking callbacks""" |
|
30 | 34 | def __init__(self, engine=None, history=None): |
|
31 | 35 | super(FrontEndCallbackChecker, self).__init__(engine=engine, |
|
32 | 36 | history=history) |
|
33 | 37 | self.updateCalled = False |
|
34 | 38 | self.renderResultCalled = False |
|
35 | 39 | self.renderErrorCalled = False |
|
36 | 40 | |
|
37 | 41 | def update_cell_prompt(self, result, blockID=None): |
|
38 | 42 | self.updateCalled = True |
|
39 | 43 | return result |
|
40 | 44 | |
|
41 | 45 | def render_result(self, result): |
|
42 | 46 | self.renderResultCalled = True |
|
43 | 47 | return result |
|
44 | 48 | |
|
45 | 49 | def render_error(self, failure): |
|
46 | 50 | self.renderErrorCalled = True |
|
47 | 51 | return failure |
|
48 | 52 | |
|
49 | 53 | |
|
50 | 54 | class TestAsyncFrontendBase(unittest.TestCase): |
|
51 | 55 | def setUp(self): |
|
52 | 56 | """Setup the EngineService and FrontEndBase""" |
|
53 | 57 | |
|
54 | 58 | self.fb = FrontEndCallbackChecker(engine=EngineService()) |
|
55 | 59 | |
|
56 | 60 | def test_implements_IFrontEnd(self): |
|
57 | 61 | self.assert_(frontendbase.IFrontEnd.implementedBy( |
|
58 | 62 | AsyncFrontEndBase)) |
|
59 | 63 | |
|
60 | 64 | def test_is_complete_returns_False_for_incomplete_block(self): |
|
61 | 65 | block = """def test(a):""" |
|
62 | 66 | self.assert_(self.fb.is_complete(block) == False) |
|
63 | 67 | |
|
64 | 68 | def test_is_complete_returns_True_for_complete_block(self): |
|
65 | 69 | block = """def test(a): pass""" |
|
66 | 70 | self.assert_(self.fb.is_complete(block)) |
|
67 | 71 | block = """a=3""" |
|
68 | 72 | self.assert_(self.fb.is_complete(block)) |
|
69 | 73 | |
|
70 | 74 | def test_blockID_added_to_result(self): |
|
71 | 75 | block = """3+3""" |
|
72 | 76 | d = self.fb.execute(block, blockID='TEST_ID') |
|
73 | 77 | d.addCallback(lambda r: self.assert_(r['blockID']=='TEST_ID')) |
|
74 | 78 | return d |
|
75 | 79 | |
|
76 | 80 | def test_blockID_added_to_failure(self): |
|
77 | 81 | block = "raise Exception()" |
|
78 | 82 | d = self.fb.execute(block,blockID='TEST_ID') |
|
79 | 83 | d.addErrback(lambda f: self.assert_(f.blockID=='TEST_ID')) |
|
80 | 84 | return d |
|
81 | 85 | |
|
82 | 86 | def test_callbacks_added_to_execute(self): |
|
83 | 87 | d = self.fb.execute("10+10") |
|
84 | 88 | d.addCallback(lambda r: self.assert_(self.fb.updateCalled and self.fb.renderResultCalled)) |
|
85 | 89 | return d |
|
86 | 90 | |
|
87 | 91 | def test_error_callback_added_to_execute(self): |
|
88 | 92 | """Test that render_error called on execution error.""" |
|
89 | 93 | |
|
90 | 94 | d = self.fb.execute("raise Exception()") |
|
91 | 95 | d.addErrback(lambda f: self.assert_(self.fb.renderErrorCalled)) |
|
92 | 96 | return d |
|
93 | 97 | |
|
94 | 98 | def test_history_returns_expected_block(self): |
|
95 | 99 | """Make sure history browsing doesn't fail.""" |
|
96 | 100 | |
|
97 | 101 | blocks = ["a=1","a=2","a=3"] |
|
98 | 102 | d = self.fb.execute(blocks[0]) |
|
99 | 103 | d.addCallback(lambda _: self.fb.execute(blocks[1])) |
|
100 | 104 | d.addCallback(lambda _: self.fb.execute(blocks[2])) |
|
101 | 105 | d.addCallback(lambda _: self.assert_(self.fb.get_history_previous("")==blocks[-2])) |
|
102 | 106 | d.addCallback(lambda _: self.assert_(self.fb.get_history_previous("")==blocks[-3])) |
|
103 | 107 | d.addCallback(lambda _: self.assert_(self.fb.get_history_next()==blocks[-2])) |
|
104 | 108 | return d |
|
105 | 109 | |
|
106 | 110 | def test_history_returns_none_at_startup(self): |
|
107 | 111 | self.assert_(self.fb.get_history_previous("")==None) |
|
108 | 112 | self.assert_(self.fb.get_history_next()==None) |
|
109 |
@@ -1,906 +1,903 b'' | |||
|
1 | 1 | # encoding: utf-8 |
|
2 | 2 | # -*- test-case-name: IPython.kernel.tests.test_engineservice -*- |
|
3 | 3 | |
|
4 | 4 | """A Twisted Service Representation of the IPython core. |
|
5 | 5 | |
|
6 | 6 | The IPython Core exposed to the network is called the Engine. Its |
|
7 | 7 | representation in Twisted in the EngineService. Interfaces and adapters |
|
8 | 8 | are used to abstract out the details of the actual network protocol used. |
|
9 | 9 | The EngineService is an Engine that knows nothing about the actual protocol |
|
10 | 10 | used. |
|
11 | 11 | |
|
12 | 12 | The EngineService is exposed with various network protocols in modules like: |
|
13 | 13 | |
|
14 | 14 | enginepb.py |
|
15 | 15 | enginevanilla.py |
|
16 | 16 | |
|
17 | 17 | As of 12/12/06 the classes in this module have been simplified greatly. It was |
|
18 | 18 | felt that we had over-engineered things. To improve the maintainability of the |
|
19 | 19 | code we have taken out the ICompleteEngine interface and the completeEngine |
|
20 | 20 | method that automatically added methods to engines. |
|
21 | 21 | |
|
22 | 22 | """ |
|
23 | 23 | |
|
24 | 24 | __docformat__ = "restructuredtext en" |
|
25 | 25 | |
|
26 | # Tell nose to skip this module. I don't think we need this as nose | |
|
27 | # shouldn't ever be run on this! | |
|
28 | __test__ = {} | |
|
29 | ||
|
26 | 30 | #------------------------------------------------------------------------------- |
|
27 | 31 | # Copyright (C) 2008 The IPython Development Team |
|
28 | 32 | # |
|
29 | 33 | # Distributed under the terms of the BSD License. The full license is in |
|
30 | 34 | # the file COPYING, distributed as part of this software. |
|
31 | 35 | #------------------------------------------------------------------------------- |
|
32 | 36 | |
|
33 | 37 | #------------------------------------------------------------------------------- |
|
34 | 38 | # Imports |
|
35 | 39 | #------------------------------------------------------------------------------- |
|
36 | 40 | |
|
37 | # Tell nose to skip the testing of this module | |
|
38 | __test__ = {} | |
|
39 | ||
|
40 | import os, sys, copy | |
|
41 | import copy | |
|
42 | import sys | |
|
41 | 43 | import cPickle as pickle |
|
42 | from new import instancemethod | |
|
43 | 44 | |
|
44 | 45 | from twisted.application import service |
|
45 | 46 | from twisted.internet import defer, reactor |
|
46 | 47 | from twisted.python import log, failure, components |
|
47 | 48 | import zope.interface as zi |
|
48 | 49 | |
|
49 | 50 | from IPython.kernel.core.interpreter import Interpreter |
|
50 |
from IPython.kernel import newserialized, error |
|
|
51 | from IPython.kernel.util import printer | |
|
52 | from IPython.kernel.twistedutil import gatherBoth, DeferredList | |
|
53 | from IPython.kernel import codeutil | |
|
54 | ||
|
51 | from IPython.kernel import newserialized, error | |
|
55 | 52 | |
|
56 | 53 | #------------------------------------------------------------------------------- |
|
57 | 54 | # Interface specification for the Engine |
|
58 | 55 | #------------------------------------------------------------------------------- |
|
59 | 56 | |
|
60 | 57 | class IEngineCore(zi.Interface): |
|
61 | 58 | """The minimal required interface for the IPython Engine. |
|
62 | 59 | |
|
63 | 60 | This interface provides a formal specification of the IPython core. |
|
64 | 61 | All these methods should return deferreds regardless of what side of a |
|
65 | 62 | network connection they are on. |
|
66 | 63 | |
|
67 | 64 | In general, this class simply wraps a shell class and wraps its return |
|
68 | 65 | values as Deferred objects. If the underlying shell class method raises |
|
69 | 66 | an exception, this class should convert it to a twisted.failure.Failure |
|
70 | 67 | that will be propagated along the Deferred's errback chain. |
|
71 | 68 | |
|
72 | 69 | In addition, Failures are aggressive. By this, we mean that if a method |
|
73 | 70 | is performing multiple actions (like pulling multiple object) if any |
|
74 | 71 | single one fails, the entire method will fail with that Failure. It is |
|
75 | 72 | all or nothing. |
|
76 | 73 | """ |
|
77 | 74 | |
|
78 | 75 | id = zi.interface.Attribute("the id of the Engine object") |
|
79 | 76 | properties = zi.interface.Attribute("A dict of properties of the Engine") |
|
80 | 77 | |
|
81 | 78 | def execute(lines): |
|
82 | 79 | """Execute lines of Python code. |
|
83 | 80 | |
|
84 | 81 | Returns a dictionary with keys (id, number, stdin, stdout, stderr) |
|
85 | 82 | upon success. |
|
86 | 83 | |
|
87 | 84 | Returns a failure object if the execution of lines raises an exception. |
|
88 | 85 | """ |
|
89 | 86 | |
|
90 | 87 | def push(namespace): |
|
91 | 88 | """Push dict namespace into the user's namespace. |
|
92 | 89 | |
|
93 | 90 | Returns a deferred to None or a failure. |
|
94 | 91 | """ |
|
95 | 92 | |
|
96 | 93 | def pull(keys): |
|
97 | 94 | """Pulls values out of the user's namespace by keys. |
|
98 | 95 | |
|
99 | 96 | Returns a deferred to a tuple objects or a single object. |
|
100 | 97 | |
|
101 | 98 | Raises NameError if any one of objects doess not exist. |
|
102 | 99 | """ |
|
103 | 100 | |
|
104 | 101 | def push_function(namespace): |
|
105 | 102 | """Push a dict of key, function pairs into the user's namespace. |
|
106 | 103 | |
|
107 | 104 | Returns a deferred to None or a failure.""" |
|
108 | 105 | |
|
109 | 106 | def pull_function(keys): |
|
110 | 107 | """Pulls functions out of the user's namespace by keys. |
|
111 | 108 | |
|
112 | 109 | Returns a deferred to a tuple of functions or a single function. |
|
113 | 110 | |
|
114 | 111 | Raises NameError if any one of the functions does not exist. |
|
115 | 112 | """ |
|
116 | 113 | |
|
117 | 114 | def get_result(i=None): |
|
118 | 115 | """Get the stdin/stdout/stderr of command i. |
|
119 | 116 | |
|
120 | 117 | Returns a deferred to a dict with keys |
|
121 | 118 | (id, number, stdin, stdout, stderr). |
|
122 | 119 | |
|
123 | 120 | Raises IndexError if command i does not exist. |
|
124 | 121 | Raises TypeError if i in not an int. |
|
125 | 122 | """ |
|
126 | 123 | |
|
127 | 124 | def reset(): |
|
128 | 125 | """Reset the shell. |
|
129 | 126 | |
|
130 | 127 | This clears the users namespace. Won't cause modules to be |
|
131 | 128 | reloaded. Should also re-initialize certain variables like id. |
|
132 | 129 | """ |
|
133 | 130 | |
|
134 | 131 | def kill(): |
|
135 | 132 | """Kill the engine by stopping the reactor.""" |
|
136 | 133 | |
|
137 | 134 | def keys(): |
|
138 | 135 | """Return the top level variables in the users namspace. |
|
139 | 136 | |
|
140 | 137 | Returns a deferred to a dict.""" |
|
141 | 138 | |
|
142 | 139 | |
|
143 | 140 | class IEngineSerialized(zi.Interface): |
|
144 | 141 | """Push/Pull methods that take Serialized objects. |
|
145 | 142 | |
|
146 | 143 | All methods should return deferreds. |
|
147 | 144 | """ |
|
148 | 145 | |
|
149 | 146 | def push_serialized(namespace): |
|
150 | 147 | """Push a dict of keys and Serialized objects into the user's namespace.""" |
|
151 | 148 | |
|
152 | 149 | def pull_serialized(keys): |
|
153 | 150 | """Pull objects by key from the user's namespace as Serialized. |
|
154 | 151 | |
|
155 | 152 | Returns a list of or one Serialized. |
|
156 | 153 | |
|
157 | 154 | Raises NameError is any one of the objects does not exist. |
|
158 | 155 | """ |
|
159 | 156 | |
|
160 | 157 | |
|
161 | 158 | class IEngineProperties(zi.Interface): |
|
162 | 159 | """Methods for access to the properties object of an Engine""" |
|
163 | 160 | |
|
164 | 161 | properties = zi.Attribute("A StrictDict object, containing the properties") |
|
165 | 162 | |
|
166 | 163 | def set_properties(properties): |
|
167 | 164 | """set properties by key and value""" |
|
168 | 165 | |
|
169 | 166 | def get_properties(keys=None): |
|
170 | 167 | """get a list of properties by `keys`, if no keys specified, get all""" |
|
171 | 168 | |
|
172 | 169 | def del_properties(keys): |
|
173 | 170 | """delete properties by `keys`""" |
|
174 | 171 | |
|
175 | 172 | def has_properties(keys): |
|
176 | 173 | """get a list of bool values for whether `properties` has `keys`""" |
|
177 | 174 | |
|
178 | 175 | def clear_properties(): |
|
179 | 176 | """clear the properties dict""" |
|
180 | 177 | |
|
181 | 178 | class IEngineBase(IEngineCore, IEngineSerialized, IEngineProperties): |
|
182 | 179 | """The basic engine interface that EngineService will implement. |
|
183 | 180 | |
|
184 | 181 | This exists so it is easy to specify adapters that adapt to and from the |
|
185 | 182 | API that the basic EngineService implements. |
|
186 | 183 | """ |
|
187 | 184 | pass |
|
188 | 185 | |
|
189 | 186 | class IEngineQueued(IEngineBase): |
|
190 | 187 | """Interface for adding a queue to an IEngineBase. |
|
191 | 188 | |
|
192 | 189 | This interface extends the IEngineBase interface to add methods for managing |
|
193 | 190 | the engine's queue. The implicit details of this interface are that the |
|
194 | 191 | execution of all methods declared in IEngineBase should appropriately be |
|
195 | 192 | put through a queue before execution. |
|
196 | 193 | |
|
197 | 194 | All methods should return deferreds. |
|
198 | 195 | """ |
|
199 | 196 | |
|
200 | 197 | def clear_queue(): |
|
201 | 198 | """Clear the queue.""" |
|
202 | 199 | |
|
203 | 200 | def queue_status(): |
|
204 | 201 | """Get the queued and pending commands in the queue.""" |
|
205 | 202 | |
|
206 | 203 | def register_failure_observer(obs): |
|
207 | 204 | """Register an observer of pending Failures. |
|
208 | 205 | |
|
209 | 206 | The observer must implement IFailureObserver. |
|
210 | 207 | """ |
|
211 | 208 | |
|
212 | 209 | def unregister_failure_observer(obs): |
|
213 | 210 | """Unregister an observer of pending Failures.""" |
|
214 | 211 | |
|
215 | 212 | |
|
216 | 213 | class IEngineThreaded(zi.Interface): |
|
217 | 214 | """A place holder for threaded commands. |
|
218 | 215 | |
|
219 | 216 | All methods should return deferreds. |
|
220 | 217 | """ |
|
221 | 218 | pass |
|
222 | 219 | |
|
223 | 220 | |
|
224 | 221 | #------------------------------------------------------------------------------- |
|
225 | 222 | # Functions and classes to implement the EngineService |
|
226 | 223 | #------------------------------------------------------------------------------- |
|
227 | 224 | |
|
228 | 225 | |
|
229 | 226 | class StrictDict(dict): |
|
230 | 227 | """This is a strict copying dictionary for use as the interface to the |
|
231 | 228 | properties of an Engine. |
|
232 | 229 | |
|
233 | 230 | :IMPORTANT: |
|
234 | 231 | This object copies the values you set to it, and returns copies to you |
|
235 | 232 | when you request them. The only way to change properties os explicitly |
|
236 | 233 | through the setitem and getitem of the dictionary interface. |
|
237 | 234 | |
|
238 | 235 | Example: |
|
239 | 236 | >>> e = get_engine(id) |
|
240 | 237 | >>> L = [1,2,3] |
|
241 | 238 | >>> e.properties['L'] = L |
|
242 | 239 | >>> L == e.properties['L'] |
|
243 | 240 | True |
|
244 | 241 | >>> L.append(99) |
|
245 | 242 | >>> L == e.properties['L'] |
|
246 | 243 | False |
|
247 | 244 | |
|
248 | 245 | Note that getitem copies, so calls to methods of objects do not affect |
|
249 | 246 | the properties, as seen here: |
|
250 | 247 | |
|
251 | 248 | >>> e.properties[1] = range(2) |
|
252 | 249 | >>> print e.properties[1] |
|
253 | 250 | [0, 1] |
|
254 | 251 | >>> e.properties[1].append(2) |
|
255 | 252 | >>> print e.properties[1] |
|
256 | 253 | [0, 1] |
|
257 | 254 | """ |
|
258 | 255 | def __init__(self, *args, **kwargs): |
|
259 | 256 | dict.__init__(self, *args, **kwargs) |
|
260 | 257 | self.modified = True |
|
261 | 258 | |
|
262 | 259 | def __getitem__(self, key): |
|
263 | 260 | return copy.deepcopy(dict.__getitem__(self, key)) |
|
264 | 261 | |
|
265 | 262 | def __setitem__(self, key, value): |
|
266 | 263 | # check if this entry is valid for transport around the network |
|
267 | 264 | # and copying |
|
268 | 265 | try: |
|
269 | 266 | pickle.dumps(key, 2) |
|
270 | 267 | pickle.dumps(value, 2) |
|
271 | 268 | newvalue = copy.deepcopy(value) |
|
272 | 269 | except Exception, e: |
|
273 | 270 | raise error.InvalidProperty("can't be a value: %r" % value) |
|
274 | 271 | dict.__setitem__(self, key, newvalue) |
|
275 | 272 | self.modified = True |
|
276 | 273 | |
|
277 | 274 | def __delitem__(self, key): |
|
278 | 275 | dict.__delitem__(self, key) |
|
279 | 276 | self.modified = True |
|
280 | 277 | |
|
281 | 278 | def update(self, dikt): |
|
282 | 279 | for k,v in dikt.iteritems(): |
|
283 | 280 | self[k] = v |
|
284 | 281 | |
|
285 | 282 | def pop(self, key): |
|
286 | 283 | self.modified = True |
|
287 | 284 | return dict.pop(self, key) |
|
288 | 285 | |
|
289 | 286 | def popitem(self): |
|
290 | 287 | self.modified = True |
|
291 | 288 | return dict.popitem(self) |
|
292 | 289 | |
|
293 | 290 | def clear(self): |
|
294 | 291 | self.modified = True |
|
295 | 292 | dict.clear(self) |
|
296 | 293 | |
|
297 | 294 | def subDict(self, *keys): |
|
298 | 295 | d = {} |
|
299 | 296 | for key in keys: |
|
300 | 297 | d[key] = self[key] |
|
301 | 298 | return d |
|
302 | 299 | |
|
303 | 300 | |
|
304 | 301 | |
|
305 | 302 | class EngineAPI(object): |
|
306 | 303 | """This is the object through which the user can edit the `properties` |
|
307 | 304 | attribute of an Engine. |
|
308 | 305 | The Engine Properties object copies all object in and out of itself. |
|
309 | 306 | See the EngineProperties object for details. |
|
310 | 307 | """ |
|
311 | 308 | _fix=False |
|
312 | 309 | def __init__(self, id): |
|
313 | 310 | self.id = id |
|
314 | 311 | self.properties = StrictDict() |
|
315 | 312 | self._fix=True |
|
316 | 313 | |
|
317 | 314 | def __setattr__(self, k,v): |
|
318 | 315 | if self._fix: |
|
319 | 316 | raise error.KernelError("I am protected!") |
|
320 | 317 | else: |
|
321 | 318 | object.__setattr__(self, k, v) |
|
322 | 319 | |
|
323 | 320 | def __delattr__(self, key): |
|
324 | 321 | raise error.KernelError("I am protected!") |
|
325 | 322 | |
|
326 | 323 | |
|
327 | 324 | _apiDict = {} |
|
328 | 325 | |
|
329 | 326 | def get_engine(id): |
|
330 | 327 | """Get the Engine API object, whcih currently just provides the properties |
|
331 | 328 | object, by ID""" |
|
332 | 329 | global _apiDict |
|
333 | 330 | if not _apiDict.get(id): |
|
334 | 331 | _apiDict[id] = EngineAPI(id) |
|
335 | 332 | return _apiDict[id] |
|
336 | 333 | |
|
337 | 334 | def drop_engine(id): |
|
338 | 335 | """remove an engine""" |
|
339 | 336 | global _apiDict |
|
340 | 337 | if _apiDict.has_key(id): |
|
341 | 338 | del _apiDict[id] |
|
342 | 339 | |
|
343 | 340 | class EngineService(object, service.Service): |
|
344 | 341 | """Adapt a IPython shell into a IEngine implementing Twisted Service.""" |
|
345 | 342 | |
|
346 | 343 | zi.implements(IEngineBase) |
|
347 | 344 | name = 'EngineService' |
|
348 | 345 | |
|
349 | 346 | def __init__(self, shellClass=Interpreter, mpi=None): |
|
350 | 347 | """Create an EngineService. |
|
351 | 348 | |
|
352 | 349 | shellClass: something that implements IInterpreter or core1 |
|
353 | 350 | mpi: an mpi module that has rank and size attributes |
|
354 | 351 | """ |
|
355 | 352 | self.shellClass = shellClass |
|
356 | 353 | self.shell = self.shellClass() |
|
357 | 354 | self.mpi = mpi |
|
358 | 355 | self.id = None |
|
359 | 356 | self.properties = get_engine(self.id).properties |
|
360 | 357 | if self.mpi is not None: |
|
361 | 358 | log.msg("MPI started with rank = %i and size = %i" % |
|
362 | 359 | (self.mpi.rank, self.mpi.size)) |
|
363 | 360 | self.id = self.mpi.rank |
|
364 | 361 | self._seedNamespace() |
|
365 | 362 | |
|
366 | 363 | # Make id a property so that the shell can get the updated id |
|
367 | 364 | |
|
368 | 365 | def _setID(self, id): |
|
369 | 366 | self._id = id |
|
370 | 367 | self.properties = get_engine(id).properties |
|
371 | 368 | self.shell.push({'id': id}) |
|
372 | 369 | |
|
373 | 370 | def _getID(self): |
|
374 | 371 | return self._id |
|
375 | 372 | |
|
376 | 373 | id = property(_getID, _setID) |
|
377 | 374 | |
|
378 | 375 | def _seedNamespace(self): |
|
379 | 376 | self.shell.push({'mpi': self.mpi, 'id' : self.id}) |
|
380 | 377 | |
|
381 | 378 | def executeAndRaise(self, msg, callable, *args, **kwargs): |
|
382 | 379 | """Call a method of self.shell and wrap any exception.""" |
|
383 | 380 | d = defer.Deferred() |
|
384 | 381 | try: |
|
385 | 382 | result = callable(*args, **kwargs) |
|
386 | 383 | except: |
|
387 | 384 | # This gives the following: |
|
388 | 385 | # et=exception class |
|
389 | 386 | # ev=exception class instance |
|
390 | 387 | # tb=traceback object |
|
391 | 388 | et,ev,tb = sys.exc_info() |
|
392 | 389 | # This call adds attributes to the exception value |
|
393 | 390 | et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg) |
|
394 | 391 | # Add another attribute |
|
395 | 392 | ev._ipython_engine_info = msg |
|
396 | 393 | f = failure.Failure(ev,et,None) |
|
397 | 394 | d.errback(f) |
|
398 | 395 | else: |
|
399 | 396 | d.callback(result) |
|
400 | 397 | |
|
401 | 398 | return d |
|
402 | 399 | |
|
403 | 400 | |
|
404 | 401 | # The IEngine methods. See the interface for documentation. |
|
405 | 402 | |
|
406 | 403 | def execute(self, lines): |
|
407 | 404 | msg = {'engineid':self.id, |
|
408 | 405 | 'method':'execute', |
|
409 | 406 | 'args':[lines]} |
|
410 | 407 | d = self.executeAndRaise(msg, self.shell.execute, lines) |
|
411 | 408 | d.addCallback(self.addIDToResult) |
|
412 | 409 | return d |
|
413 | 410 | |
|
414 | 411 | def addIDToResult(self, result): |
|
415 | 412 | result['id'] = self.id |
|
416 | 413 | return result |
|
417 | 414 | |
|
418 | 415 | def push(self, namespace): |
|
419 | 416 | msg = {'engineid':self.id, |
|
420 | 417 | 'method':'push', |
|
421 | 418 | 'args':[repr(namespace.keys())]} |
|
422 | 419 | d = self.executeAndRaise(msg, self.shell.push, namespace) |
|
423 | 420 | return d |
|
424 | 421 | |
|
425 | 422 | def pull(self, keys): |
|
426 | 423 | msg = {'engineid':self.id, |
|
427 | 424 | 'method':'pull', |
|
428 | 425 | 'args':[repr(keys)]} |
|
429 | 426 | d = self.executeAndRaise(msg, self.shell.pull, keys) |
|
430 | 427 | return d |
|
431 | 428 | |
|
432 | 429 | def push_function(self, namespace): |
|
433 | 430 | msg = {'engineid':self.id, |
|
434 | 431 | 'method':'push_function', |
|
435 | 432 | 'args':[repr(namespace.keys())]} |
|
436 | 433 | d = self.executeAndRaise(msg, self.shell.push_function, namespace) |
|
437 | 434 | return d |
|
438 | 435 | |
|
439 | 436 | def pull_function(self, keys): |
|
440 | 437 | msg = {'engineid':self.id, |
|
441 | 438 | 'method':'pull_function', |
|
442 | 439 | 'args':[repr(keys)]} |
|
443 | 440 | d = self.executeAndRaise(msg, self.shell.pull_function, keys) |
|
444 | 441 | return d |
|
445 | 442 | |
|
446 | 443 | def get_result(self, i=None): |
|
447 | 444 | msg = {'engineid':self.id, |
|
448 | 445 | 'method':'get_result', |
|
449 | 446 | 'args':[repr(i)]} |
|
450 | 447 | d = self.executeAndRaise(msg, self.shell.getCommand, i) |
|
451 | 448 | d.addCallback(self.addIDToResult) |
|
452 | 449 | return d |
|
453 | 450 | |
|
454 | 451 | def reset(self): |
|
455 | 452 | msg = {'engineid':self.id, |
|
456 | 453 | 'method':'reset', |
|
457 | 454 | 'args':[]} |
|
458 | 455 | del self.shell |
|
459 | 456 | self.shell = self.shellClass() |
|
460 | 457 | self.properties.clear() |
|
461 | 458 | d = self.executeAndRaise(msg, self._seedNamespace) |
|
462 | 459 | return d |
|
463 | 460 | |
|
464 | 461 | def kill(self): |
|
465 | 462 | drop_engine(self.id) |
|
466 | 463 | try: |
|
467 | 464 | reactor.stop() |
|
468 | 465 | except RuntimeError: |
|
469 | 466 | log.msg('The reactor was not running apparently.') |
|
470 | 467 | return defer.fail() |
|
471 | 468 | else: |
|
472 | 469 | return defer.succeed(None) |
|
473 | 470 | |
|
474 | 471 | def keys(self): |
|
475 | 472 | """Return a list of variables names in the users top level namespace. |
|
476 | 473 | |
|
477 | 474 | This used to return a dict of all the keys/repr(values) in the |
|
478 | 475 | user's namespace. This was too much info for the ControllerService |
|
479 | 476 | to handle so it is now just a list of keys. |
|
480 | 477 | """ |
|
481 | 478 | |
|
482 | 479 | remotes = [] |
|
483 | 480 | for k in self.shell.user_ns.iterkeys(): |
|
484 | 481 | if k not in ['__name__', '_ih', '_oh', '__builtins__', |
|
485 | 482 | 'In', 'Out', '_', '__', '___', '__IP', 'input', 'raw_input']: |
|
486 | 483 | remotes.append(k) |
|
487 | 484 | return defer.succeed(remotes) |
|
488 | 485 | |
|
489 | 486 | def set_properties(self, properties): |
|
490 | 487 | msg = {'engineid':self.id, |
|
491 | 488 | 'method':'set_properties', |
|
492 | 489 | 'args':[repr(properties.keys())]} |
|
493 | 490 | return self.executeAndRaise(msg, self.properties.update, properties) |
|
494 | 491 | |
|
495 | 492 | def get_properties(self, keys=None): |
|
496 | 493 | msg = {'engineid':self.id, |
|
497 | 494 | 'method':'get_properties', |
|
498 | 495 | 'args':[repr(keys)]} |
|
499 | 496 | if keys is None: |
|
500 | 497 | keys = self.properties.keys() |
|
501 | 498 | return self.executeAndRaise(msg, self.properties.subDict, *keys) |
|
502 | 499 | |
|
503 | 500 | def _doDel(self, keys): |
|
504 | 501 | for key in keys: |
|
505 | 502 | del self.properties[key] |
|
506 | 503 | |
|
507 | 504 | def del_properties(self, keys): |
|
508 | 505 | msg = {'engineid':self.id, |
|
509 | 506 | 'method':'del_properties', |
|
510 | 507 | 'args':[repr(keys)]} |
|
511 | 508 | return self.executeAndRaise(msg, self._doDel, keys) |
|
512 | 509 | |
|
513 | 510 | def _doHas(self, keys): |
|
514 | 511 | return [self.properties.has_key(key) for key in keys] |
|
515 | 512 | |
|
516 | 513 | def has_properties(self, keys): |
|
517 | 514 | msg = {'engineid':self.id, |
|
518 | 515 | 'method':'has_properties', |
|
519 | 516 | 'args':[repr(keys)]} |
|
520 | 517 | return self.executeAndRaise(msg, self._doHas, keys) |
|
521 | 518 | |
|
522 | 519 | def clear_properties(self): |
|
523 | 520 | msg = {'engineid':self.id, |
|
524 | 521 | 'method':'clear_properties', |
|
525 | 522 | 'args':[]} |
|
526 | 523 | return self.executeAndRaise(msg, self.properties.clear) |
|
527 | 524 | |
|
528 | 525 | def push_serialized(self, sNamespace): |
|
529 | 526 | msg = {'engineid':self.id, |
|
530 | 527 | 'method':'push_serialized', |
|
531 | 528 | 'args':[repr(sNamespace.keys())]} |
|
532 | 529 | ns = {} |
|
533 | 530 | for k,v in sNamespace.iteritems(): |
|
534 | 531 | try: |
|
535 | 532 | unserialized = newserialized.IUnSerialized(v) |
|
536 | 533 | ns[k] = unserialized.getObject() |
|
537 | 534 | except: |
|
538 | 535 | return defer.fail() |
|
539 | 536 | return self.executeAndRaise(msg, self.shell.push, ns) |
|
540 | 537 | |
|
541 | 538 | def pull_serialized(self, keys): |
|
542 | 539 | msg = {'engineid':self.id, |
|
543 | 540 | 'method':'pull_serialized', |
|
544 | 541 | 'args':[repr(keys)]} |
|
545 | 542 | if isinstance(keys, str): |
|
546 | 543 | keys = [keys] |
|
547 | 544 | if len(keys)==1: |
|
548 | 545 | d = self.executeAndRaise(msg, self.shell.pull, keys) |
|
549 | 546 | d.addCallback(newserialized.serialize) |
|
550 | 547 | return d |
|
551 | 548 | elif len(keys)>1: |
|
552 | 549 | d = self.executeAndRaise(msg, self.shell.pull, keys) |
|
553 | 550 | @d.addCallback |
|
554 | 551 | def packThemUp(values): |
|
555 | 552 | serials = [] |
|
556 | 553 | for v in values: |
|
557 | 554 | try: |
|
558 | 555 | serials.append(newserialized.serialize(v)) |
|
559 | 556 | except: |
|
560 | 557 | return defer.fail(failure.Failure()) |
|
561 | 558 | return serials |
|
562 | 559 | return packThemUp |
|
563 | 560 | |
|
564 | 561 | |
|
565 | 562 | def queue(methodToQueue): |
|
566 | 563 | def queuedMethod(this, *args, **kwargs): |
|
567 | 564 | name = methodToQueue.__name__ |
|
568 | 565 | return this.submitCommand(Command(name, *args, **kwargs)) |
|
569 | 566 | return queuedMethod |
|
570 | 567 | |
|
571 | 568 | class QueuedEngine(object): |
|
572 | 569 | """Adapt an IEngineBase to an IEngineQueued by wrapping it. |
|
573 | 570 | |
|
574 | 571 | The resulting object will implement IEngineQueued which extends |
|
575 | 572 | IEngineCore which extends (IEngineBase, IEngineSerialized). |
|
576 | 573 | |
|
577 | 574 | This seems like the best way of handling it, but I am not sure. The |
|
578 | 575 | other option is to have the various base interfaces be used like |
|
579 | 576 | mix-in intefaces. The problem I have with this is adpatation is |
|
580 | 577 | more difficult and complicated because there can be can multiple |
|
581 | 578 | original and final Interfaces. |
|
582 | 579 | """ |
|
583 | 580 | |
|
584 | 581 | zi.implements(IEngineQueued) |
|
585 | 582 | |
|
586 | 583 | def __init__(self, engine): |
|
587 | 584 | """Create a QueuedEngine object from an engine |
|
588 | 585 | |
|
589 | 586 | engine: An implementor of IEngineCore and IEngineSerialized |
|
590 | 587 | keepUpToDate: whether to update the remote status when the |
|
591 | 588 | queue is empty. Defaults to False. |
|
592 | 589 | """ |
|
593 | 590 | |
|
594 | 591 | # This is the right way to do these tests rather than |
|
595 | 592 | # IEngineCore in list(zi.providedBy(engine)) which will only |
|
596 | 593 | # picks of the interfaces that are directly declared by engine. |
|
597 | 594 | assert IEngineBase.providedBy(engine), \ |
|
598 | 595 | "engine passed to QueuedEngine doesn't provide IEngineBase" |
|
599 | 596 | |
|
600 | 597 | self.engine = engine |
|
601 | 598 | self.id = engine.id |
|
602 | 599 | self.queued = [] |
|
603 | 600 | self.history = {} |
|
604 | 601 | self.engineStatus = {} |
|
605 | 602 | self.currentCommand = None |
|
606 | 603 | self.failureObservers = [] |
|
607 | 604 | |
|
608 | 605 | def _get_properties(self): |
|
609 | 606 | return self.engine.properties |
|
610 | 607 | |
|
611 | 608 | properties = property(_get_properties, lambda self, _: None) |
|
612 | 609 | # Queue management methods. You should not call these directly |
|
613 | 610 | |
|
614 | 611 | def submitCommand(self, cmd): |
|
615 | 612 | """Submit command to queue.""" |
|
616 | 613 | |
|
617 | 614 | d = defer.Deferred() |
|
618 | 615 | cmd.setDeferred(d) |
|
619 | 616 | if self.currentCommand is not None: |
|
620 | 617 | if self.currentCommand.finished: |
|
621 | 618 | # log.msg("Running command immediately: %r" % cmd) |
|
622 | 619 | self.currentCommand = cmd |
|
623 | 620 | self.runCurrentCommand() |
|
624 | 621 | else: # command is still running |
|
625 | 622 | # log.msg("Command is running: %r" % self.currentCommand) |
|
626 | 623 | # log.msg("Queueing: %r" % cmd) |
|
627 | 624 | self.queued.append(cmd) |
|
628 | 625 | else: |
|
629 | 626 | # log.msg("No current commands, running: %r" % cmd) |
|
630 | 627 | self.currentCommand = cmd |
|
631 | 628 | self.runCurrentCommand() |
|
632 | 629 | return d |
|
633 | 630 | |
|
634 | 631 | def runCurrentCommand(self): |
|
635 | 632 | """Run current command.""" |
|
636 | 633 | |
|
637 | 634 | cmd = self.currentCommand |
|
638 | 635 | f = getattr(self.engine, cmd.remoteMethod, None) |
|
639 | 636 | if f: |
|
640 | 637 | d = f(*cmd.args, **cmd.kwargs) |
|
641 | 638 | if cmd.remoteMethod is 'execute': |
|
642 | 639 | d.addCallback(self.saveResult) |
|
643 | 640 | d.addCallback(self.finishCommand) |
|
644 | 641 | d.addErrback(self.abortCommand) |
|
645 | 642 | else: |
|
646 | 643 | return defer.fail(AttributeError(cmd.remoteMethod)) |
|
647 | 644 | |
|
648 | 645 | def _flushQueue(self): |
|
649 | 646 | """Pop next command in queue and run it.""" |
|
650 | 647 | |
|
651 | 648 | if len(self.queued) > 0: |
|
652 | 649 | self.currentCommand = self.queued.pop(0) |
|
653 | 650 | self.runCurrentCommand() |
|
654 | 651 | |
|
655 | 652 | def saveResult(self, result): |
|
656 | 653 | """Put the result in the history.""" |
|
657 | 654 | self.history[result['number']] = result |
|
658 | 655 | return result |
|
659 | 656 | |
|
660 | 657 | def finishCommand(self, result): |
|
661 | 658 | """Finish currrent command.""" |
|
662 | 659 | |
|
663 | 660 | # The order of these commands is absolutely critical. |
|
664 | 661 | self.currentCommand.handleResult(result) |
|
665 | 662 | self.currentCommand.finished = True |
|
666 | 663 | self._flushQueue() |
|
667 | 664 | return result |
|
668 | 665 | |
|
669 | 666 | def abortCommand(self, reason): |
|
670 | 667 | """Abort current command. |
|
671 | 668 | |
|
672 | 669 | This eats the Failure but first passes it onto the Deferred that the |
|
673 | 670 | user has. |
|
674 | 671 | |
|
675 | 672 | It also clear out the queue so subsequence commands don't run. |
|
676 | 673 | """ |
|
677 | 674 | |
|
678 | 675 | # The order of these 3 commands is absolutely critical. The currentCommand |
|
679 | 676 | # must first be marked as finished BEFORE the queue is cleared and before |
|
680 | 677 | # the current command is sent the failure. |
|
681 | 678 | # Also, the queue must be cleared BEFORE the current command is sent the Failure |
|
682 | 679 | # otherwise the errback chain could trigger new commands to be added to the |
|
683 | 680 | # queue before we clear it. We should clear ONLY the commands that were in |
|
684 | 681 | # the queue when the error occured. |
|
685 | 682 | self.currentCommand.finished = True |
|
686 | 683 | s = "%r %r %r" % (self.currentCommand.remoteMethod, self.currentCommand.args, self.currentCommand.kwargs) |
|
687 | 684 | self.clear_queue(msg=s) |
|
688 | 685 | self.currentCommand.handleError(reason) |
|
689 | 686 | |
|
690 | 687 | return None |
|
691 | 688 | |
|
692 | 689 | #--------------------------------------------------------------------------- |
|
693 | 690 | # IEngineCore methods |
|
694 | 691 | #--------------------------------------------------------------------------- |
|
695 | 692 | |
|
696 | 693 | @queue |
|
697 | 694 | def execute(self, lines): |
|
698 | 695 | pass |
|
699 | 696 | |
|
700 | 697 | @queue |
|
701 | 698 | def push(self, namespace): |
|
702 | 699 | pass |
|
703 | 700 | |
|
704 | 701 | @queue |
|
705 | 702 | def pull(self, keys): |
|
706 | 703 | pass |
|
707 | 704 | |
|
708 | 705 | @queue |
|
709 | 706 | def push_function(self, namespace): |
|
710 | 707 | pass |
|
711 | 708 | |
|
712 | 709 | @queue |
|
713 | 710 | def pull_function(self, keys): |
|
714 | 711 | pass |
|
715 | 712 | |
|
716 | 713 | def get_result(self, i=None): |
|
717 | 714 | if i is None: |
|
718 | 715 | i = max(self.history.keys()+[None]) |
|
719 | 716 | |
|
720 | 717 | cmd = self.history.get(i, None) |
|
721 | 718 | # Uncomment this line to disable chaching of results |
|
722 | 719 | #cmd = None |
|
723 | 720 | if cmd is None: |
|
724 | 721 | return self.submitCommand(Command('get_result', i)) |
|
725 | 722 | else: |
|
726 | 723 | return defer.succeed(cmd) |
|
727 | 724 | |
|
728 | 725 | def reset(self): |
|
729 | 726 | self.clear_queue() |
|
730 | 727 | self.history = {} # reset the cache - I am not sure we should do this |
|
731 | 728 | return self.submitCommand(Command('reset')) |
|
732 | 729 | |
|
733 | 730 | def kill(self): |
|
734 | 731 | self.clear_queue() |
|
735 | 732 | return self.submitCommand(Command('kill')) |
|
736 | 733 | |
|
737 | 734 | @queue |
|
738 | 735 | def keys(self): |
|
739 | 736 | pass |
|
740 | 737 | |
|
741 | 738 | #--------------------------------------------------------------------------- |
|
742 | 739 | # IEngineSerialized methods |
|
743 | 740 | #--------------------------------------------------------------------------- |
|
744 | 741 | |
|
745 | 742 | @queue |
|
746 | 743 | def push_serialized(self, namespace): |
|
747 | 744 | pass |
|
748 | 745 | |
|
749 | 746 | @queue |
|
750 | 747 | def pull_serialized(self, keys): |
|
751 | 748 | pass |
|
752 | 749 | |
|
753 | 750 | #--------------------------------------------------------------------------- |
|
754 | 751 | # IEngineProperties methods |
|
755 | 752 | #--------------------------------------------------------------------------- |
|
756 | 753 | |
|
757 | 754 | @queue |
|
758 | 755 | def set_properties(self, namespace): |
|
759 | 756 | pass |
|
760 | 757 | |
|
761 | 758 | @queue |
|
762 | 759 | def get_properties(self, keys=None): |
|
763 | 760 | pass |
|
764 | 761 | |
|
765 | 762 | @queue |
|
766 | 763 | def del_properties(self, keys): |
|
767 | 764 | pass |
|
768 | 765 | |
|
769 | 766 | @queue |
|
770 | 767 | def has_properties(self, keys): |
|
771 | 768 | pass |
|
772 | 769 | |
|
773 | 770 | @queue |
|
774 | 771 | def clear_properties(self): |
|
775 | 772 | pass |
|
776 | 773 | |
|
777 | 774 | #--------------------------------------------------------------------------- |
|
778 | 775 | # IQueuedEngine methods |
|
779 | 776 | #--------------------------------------------------------------------------- |
|
780 | 777 | |
|
781 | 778 | def clear_queue(self, msg=''): |
|
782 | 779 | """Clear the queue, but doesn't cancel the currently running commmand.""" |
|
783 | 780 | |
|
784 | 781 | for cmd in self.queued: |
|
785 | 782 | cmd.deferred.errback(failure.Failure(error.QueueCleared(msg))) |
|
786 | 783 | self.queued = [] |
|
787 | 784 | return defer.succeed(None) |
|
788 | 785 | |
|
789 | 786 | def queue_status(self): |
|
790 | 787 | if self.currentCommand is not None: |
|
791 | 788 | if self.currentCommand.finished: |
|
792 | 789 | pending = repr(None) |
|
793 | 790 | else: |
|
794 | 791 | pending = repr(self.currentCommand) |
|
795 | 792 | else: |
|
796 | 793 | pending = repr(None) |
|
797 | 794 | dikt = {'queue':map(repr,self.queued), 'pending':pending} |
|
798 | 795 | return defer.succeed(dikt) |
|
799 | 796 | |
|
800 | 797 | def register_failure_observer(self, obs): |
|
801 | 798 | self.failureObservers.append(obs) |
|
802 | 799 | |
|
803 | 800 | def unregister_failure_observer(self, obs): |
|
804 | 801 | self.failureObservers.remove(obs) |
|
805 | 802 | |
|
806 | 803 | |
|
807 | 804 | # Now register QueuedEngine as an adpater class that makes an IEngineBase into a |
|
808 | 805 | # IEngineQueued. |
|
809 | 806 | components.registerAdapter(QueuedEngine, IEngineBase, IEngineQueued) |
|
810 | 807 | |
|
811 | 808 | |
|
812 | 809 | class Command(object): |
|
813 | 810 | """A command object that encapslates queued commands. |
|
814 | 811 | |
|
815 | 812 | This class basically keeps track of a command that has been queued |
|
816 | 813 | in a QueuedEngine. It manages the deferreds and hold the method to be called |
|
817 | 814 | and the arguments to that method. |
|
818 | 815 | """ |
|
819 | 816 | |
|
820 | 817 | |
|
821 | 818 | def __init__(self, remoteMethod, *args, **kwargs): |
|
822 | 819 | """Build a new Command object.""" |
|
823 | 820 | |
|
824 | 821 | self.remoteMethod = remoteMethod |
|
825 | 822 | self.args = args |
|
826 | 823 | self.kwargs = kwargs |
|
827 | 824 | self.finished = False |
|
828 | 825 | |
|
829 | 826 | def setDeferred(self, d): |
|
830 | 827 | """Sets the deferred attribute of the Command.""" |
|
831 | 828 | |
|
832 | 829 | self.deferred = d |
|
833 | 830 | |
|
834 | 831 | def __repr__(self): |
|
835 | 832 | if not self.args: |
|
836 | 833 | args = '' |
|
837 | 834 | else: |
|
838 | 835 | args = str(self.args)[1:-2] #cut off (...,) |
|
839 | 836 | for k,v in self.kwargs.iteritems(): |
|
840 | 837 | if args: |
|
841 | 838 | args += ', ' |
|
842 | 839 | args += '%s=%r' %(k,v) |
|
843 | 840 | return "%s(%s)" %(self.remoteMethod, args) |
|
844 | 841 | |
|
845 | 842 | def handleResult(self, result): |
|
846 | 843 | """When the result is ready, relay it to self.deferred.""" |
|
847 | 844 | |
|
848 | 845 | self.deferred.callback(result) |
|
849 | 846 | |
|
850 | 847 | def handleError(self, reason): |
|
851 | 848 | """When an error has occured, relay it to self.deferred.""" |
|
852 | 849 | |
|
853 | 850 | self.deferred.errback(reason) |
|
854 | 851 | |
|
855 | 852 | class ThreadedEngineService(EngineService): |
|
856 | 853 | """An EngineService subclass that defers execute commands to a separate |
|
857 | 854 | thread. |
|
858 | 855 | |
|
859 | 856 | ThreadedEngineService uses twisted.internet.threads.deferToThread to |
|
860 | 857 | defer execute requests to a separate thread. GUI frontends may want to |
|
861 | 858 | use ThreadedEngineService as the engine in an |
|
862 | 859 | IPython.frontend.frontendbase.FrontEndBase subclass to prevent |
|
863 | 860 | block execution from blocking the GUI thread. |
|
864 | 861 | """ |
|
865 | 862 | |
|
866 | 863 | zi.implements(IEngineBase) |
|
867 | 864 | |
|
868 | 865 | def __init__(self, shellClass=Interpreter, mpi=None): |
|
869 | 866 | EngineService.__init__(self, shellClass, mpi) |
|
870 | 867 | |
|
871 | 868 | def wrapped_execute(self, msg, lines): |
|
872 | 869 | """Wrap self.shell.execute to add extra information to tracebacks""" |
|
873 | 870 | |
|
874 | 871 | try: |
|
875 | 872 | result = self.shell.execute(lines) |
|
876 | 873 | except Exception,e: |
|
877 | 874 | # This gives the following: |
|
878 | 875 | # et=exception class |
|
879 | 876 | # ev=exception class instance |
|
880 | 877 | # tb=traceback object |
|
881 | 878 | et,ev,tb = sys.exc_info() |
|
882 | 879 | # This call adds attributes to the exception value |
|
883 | 880 | et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg) |
|
884 | 881 | # Add another attribute |
|
885 | 882 | |
|
886 | 883 | # Create a new exception with the new attributes |
|
887 | 884 | e = et(ev._ipython_traceback_text) |
|
888 | 885 | e._ipython_engine_info = msg |
|
889 | 886 | |
|
890 | 887 | # Re-raise |
|
891 | 888 | raise e |
|
892 | 889 | |
|
893 | 890 | return result |
|
894 | 891 | |
|
895 | 892 | |
|
896 | 893 | def execute(self, lines): |
|
897 | 894 | # Only import this if we are going to use this class |
|
898 | 895 | from twisted.internet import threads |
|
899 | 896 | |
|
900 | 897 | msg = {'engineid':self.id, |
|
901 | 898 | 'method':'execute', |
|
902 | 899 | 'args':[lines]} |
|
903 | 900 | |
|
904 | 901 | d = threads.deferToThread(self.wrapped_execute, msg, lines) |
|
905 | 902 | d.addCallback(self.addIDToResult) |
|
906 | 903 | return d |
@@ -1,202 +1,205 b'' | |||
|
1 | 1 | # encoding: utf-8 |
|
2 | 2 | |
|
3 | 3 | """Classes and functions for kernel related errors and exceptions.""" |
|
4 | 4 | |
|
5 | 5 | __docformat__ = "restructuredtext en" |
|
6 | 6 | |
|
7 | # Tell nose to skip this module | |
|
8 | __test__ = {} | |
|
9 | ||
|
7 | 10 | #------------------------------------------------------------------------------- |
|
8 | 11 | # Copyright (C) 2008 The IPython Development Team |
|
9 | 12 | # |
|
10 | 13 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | 14 | # the file COPYING, distributed as part of this software. |
|
12 | 15 | #------------------------------------------------------------------------------- |
|
13 | 16 | |
|
14 | 17 | #------------------------------------------------------------------------------- |
|
15 | 18 | # Imports |
|
16 | 19 | #------------------------------------------------------------------------------- |
|
20 | from twisted.python import failure | |
|
17 | 21 | |
|
18 | 22 | from IPython.kernel.core import error |
|
19 | from twisted.python import failure | |
|
20 | 23 | |
|
21 | 24 | #------------------------------------------------------------------------------- |
|
22 | 25 | # Error classes |
|
23 | 26 | #------------------------------------------------------------------------------- |
|
24 | 27 | |
|
25 | 28 | class KernelError(error.IPythonError): |
|
26 | 29 | pass |
|
27 | 30 | |
|
28 | 31 | class NotDefined(KernelError): |
|
29 | 32 | def __init__(self, name): |
|
30 | 33 | self.name = name |
|
31 | 34 | self.args = (name,) |
|
32 | 35 | |
|
33 | 36 | def __repr__(self): |
|
34 | 37 | return '<NotDefined: %s>' % self.name |
|
35 | 38 | |
|
36 | 39 | __str__ = __repr__ |
|
37 | 40 | |
|
38 | 41 | class QueueCleared(KernelError): |
|
39 | 42 | pass |
|
40 | 43 | |
|
41 | 44 | class IdInUse(KernelError): |
|
42 | 45 | pass |
|
43 | 46 | |
|
44 | 47 | class ProtocolError(KernelError): |
|
45 | 48 | pass |
|
46 | 49 | |
|
47 | 50 | class ConnectionError(KernelError): |
|
48 | 51 | pass |
|
49 | 52 | |
|
50 | 53 | class InvalidEngineID(KernelError): |
|
51 | 54 | pass |
|
52 | 55 | |
|
53 | 56 | class NoEnginesRegistered(KernelError): |
|
54 | 57 | pass |
|
55 | 58 | |
|
56 | 59 | class InvalidClientID(KernelError): |
|
57 | 60 | pass |
|
58 | 61 | |
|
59 | 62 | class InvalidDeferredID(KernelError): |
|
60 | 63 | pass |
|
61 | 64 | |
|
62 | 65 | class SerializationError(KernelError): |
|
63 | 66 | pass |
|
64 | 67 | |
|
65 | 68 | class MessageSizeError(KernelError): |
|
66 | 69 | pass |
|
67 | 70 | |
|
68 | 71 | class PBMessageSizeError(MessageSizeError): |
|
69 | 72 | pass |
|
70 | 73 | |
|
71 | 74 | class ResultNotCompleted(KernelError): |
|
72 | 75 | pass |
|
73 | 76 | |
|
74 | 77 | class ResultAlreadyRetrieved(KernelError): |
|
75 | 78 | pass |
|
76 | 79 | |
|
77 | 80 | class ClientError(KernelError): |
|
78 | 81 | pass |
|
79 | 82 | |
|
80 | 83 | class TaskAborted(KernelError): |
|
81 | 84 | pass |
|
82 | 85 | |
|
83 | 86 | class TaskTimeout(KernelError): |
|
84 | 87 | pass |
|
85 | 88 | |
|
86 | 89 | class NotAPendingResult(KernelError): |
|
87 | 90 | pass |
|
88 | 91 | |
|
89 | 92 | class UnpickleableException(KernelError): |
|
90 | 93 | pass |
|
91 | 94 | |
|
92 | 95 | class AbortedPendingDeferredError(KernelError): |
|
93 | 96 | pass |
|
94 | 97 | |
|
95 | 98 | class InvalidProperty(KernelError): |
|
96 | 99 | pass |
|
97 | 100 | |
|
98 | 101 | class MissingBlockArgument(KernelError): |
|
99 | 102 | pass |
|
100 | 103 | |
|
101 | 104 | class StopLocalExecution(KernelError): |
|
102 | 105 | pass |
|
103 | 106 | |
|
104 | 107 | class SecurityError(KernelError): |
|
105 | 108 | pass |
|
106 | 109 | |
|
107 | 110 | class FileTimeoutError(KernelError): |
|
108 | 111 | pass |
|
109 | 112 | |
|
110 | 113 | class TaskRejectError(KernelError): |
|
111 | 114 | """Exception to raise when a task should be rejected by an engine. |
|
112 | 115 | |
|
113 | 116 | This exception can be used to allow a task running on an engine to test |
|
114 | 117 | if the engine (or the user's namespace on the engine) has the needed |
|
115 | 118 | task dependencies. If not, the task should raise this exception. For |
|
116 | 119 | the task to be retried on another engine, the task should be created |
|
117 | 120 | with the `retries` argument > 1. |
|
118 | 121 | |
|
119 | 122 | The advantage of this approach over our older properties system is that |
|
120 | 123 | tasks have full access to the user's namespace on the engines and the |
|
121 | 124 | properties don't have to be managed or tested by the controller. |
|
122 | 125 | """ |
|
123 | 126 | |
|
124 | 127 | class CompositeError(KernelError): |
|
125 | 128 | def __init__(self, message, elist): |
|
126 | 129 | Exception.__init__(self, *(message, elist)) |
|
127 | 130 | self.message = message |
|
128 | 131 | self.elist = elist |
|
129 | 132 | |
|
130 | 133 | def _get_engine_str(self, ev): |
|
131 | 134 | try: |
|
132 | 135 | ei = ev._ipython_engine_info |
|
133 | 136 | except AttributeError: |
|
134 | 137 | return '[Engine Exception]' |
|
135 | 138 | else: |
|
136 | 139 | return '[%i:%s]: ' % (ei['engineid'], ei['method']) |
|
137 | 140 | |
|
138 | 141 | def _get_traceback(self, ev): |
|
139 | 142 | try: |
|
140 | 143 | tb = ev._ipython_traceback_text |
|
141 | 144 | except AttributeError: |
|
142 | 145 | return 'No traceback available' |
|
143 | 146 | else: |
|
144 | 147 | return tb |
|
145 | 148 | |
|
146 | 149 | def __str__(self): |
|
147 | 150 | s = str(self.message) |
|
148 | 151 | for et, ev, etb in self.elist: |
|
149 | 152 | engine_str = self._get_engine_str(ev) |
|
150 | 153 | s = s + '\n' + engine_str + str(et.__name__) + ': ' + str(ev) |
|
151 | 154 | return s |
|
152 | 155 | |
|
153 | 156 | def print_tracebacks(self, excid=None): |
|
154 | 157 | if excid is None: |
|
155 | 158 | for (et,ev,etb) in self.elist: |
|
156 | 159 | print self._get_engine_str(ev) |
|
157 | 160 | print self._get_traceback(ev) |
|
158 | 161 | |
|
159 | 162 | else: |
|
160 | 163 | try: |
|
161 | 164 | et,ev,etb = self.elist[excid] |
|
162 | 165 | except: |
|
163 | 166 | raise IndexError("an exception with index %i does not exist"%excid) |
|
164 | 167 | else: |
|
165 | 168 | print self._get_engine_str(ev) |
|
166 | 169 | print self._get_traceback(ev) |
|
167 | 170 | |
|
168 | 171 | def raise_exception(self, excid=0): |
|
169 | 172 | try: |
|
170 | 173 | et,ev,etb = self.elist[excid] |
|
171 | 174 | except: |
|
172 | 175 | raise IndexError("an exception with index %i does not exist"%excid) |
|
173 | 176 | else: |
|
174 | 177 | raise et, ev, etb |
|
175 | 178 | |
|
176 | 179 | def collect_exceptions(rlist, method): |
|
177 | 180 | elist = [] |
|
178 | 181 | for r in rlist: |
|
179 | 182 | if isinstance(r, failure.Failure): |
|
180 | 183 | r.cleanFailure() |
|
181 | 184 | et, ev, etb = r.type, r.value, r.tb |
|
182 | 185 | # Sometimes we could have CompositeError in our list. Just take |
|
183 | 186 | # the errors out of them and put them in our new list. This |
|
184 | 187 | # has the effect of flattening lists of CompositeErrors into one |
|
185 | 188 | # CompositeError |
|
186 | 189 | if et==CompositeError: |
|
187 | 190 | for e in ev.elist: |
|
188 | 191 | elist.append(e) |
|
189 | 192 | else: |
|
190 | 193 | elist.append((et, ev, etb)) |
|
191 | 194 | if len(elist)==0: |
|
192 | 195 | return rlist |
|
193 | 196 | else: |
|
194 | 197 | msg = "one or more exceptions from call to method: %s" % (method) |
|
195 | 198 | # This silliness is needed so the debugger has access to the exception |
|
196 | 199 | # instance (e in this case) |
|
197 | 200 | try: |
|
198 | 201 | raise CompositeError(msg, elist) |
|
199 | 202 | except CompositeError, e: |
|
200 | 203 | raise e |
|
201 | 204 | |
|
202 | 205 |
@@ -1,163 +1,170 b'' | |||
|
1 | 1 | # encoding: utf-8 |
|
2 | 2 | # -*- test-case-name: IPython.kernel.test.test_newserialized -*- |
|
3 | 3 | |
|
4 | 4 | """Refactored serialization classes and interfaces.""" |
|
5 | 5 | |
|
6 | 6 | __docformat__ = "restructuredtext en" |
|
7 | 7 | |
|
8 | # Tell nose to skip this module | |
|
9 | __test__ = {} | |
|
10 | ||
|
8 | 11 | #------------------------------------------------------------------------------- |
|
9 | 12 | # Copyright (C) 2008 The IPython Development Team |
|
10 | 13 | # |
|
11 | 14 | # Distributed under the terms of the BSD License. The full license is in |
|
12 | 15 | # the file COPYING, distributed as part of this software. |
|
13 | 16 | #------------------------------------------------------------------------------- |
|
14 | 17 | |
|
15 | 18 | #------------------------------------------------------------------------------- |
|
16 | 19 | # Imports |
|
17 | 20 | #------------------------------------------------------------------------------- |
|
18 | 21 | |
|
19 | 22 | import cPickle as pickle |
|
20 | 23 | |
|
21 | from zope.interface import Interface, implements | |
|
22 | 24 | from twisted.python import components |
|
25 | from zope.interface import Interface, implements | |
|
23 | 26 | |
|
24 | 27 | try: |
|
25 | 28 | import numpy |
|
26 | 29 | except ImportError: |
|
27 | 30 | pass |
|
28 | 31 | |
|
29 | 32 | from IPython.kernel.error import SerializationError |
|
30 | 33 | |
|
34 | #----------------------------------------------------------------------------- | |
|
35 | # Classes and functions | |
|
36 | #----------------------------------------------------------------------------- | |
|
37 | ||
|
31 | 38 | class ISerialized(Interface): |
|
32 | 39 | |
|
33 | 40 | def getData(): |
|
34 | 41 | """""" |
|
35 | 42 | |
|
36 | 43 | def getDataSize(units=10.0**6): |
|
37 | 44 | """""" |
|
38 | 45 | |
|
39 | 46 | def getTypeDescriptor(): |
|
40 | 47 | """""" |
|
41 | 48 | |
|
42 | 49 | def getMetadata(): |
|
43 | 50 | """""" |
|
44 | 51 | |
|
45 | 52 | |
|
46 | 53 | class IUnSerialized(Interface): |
|
47 | 54 | |
|
48 | 55 | def getObject(): |
|
49 | 56 | """""" |
|
50 | 57 | |
|
51 | 58 | class Serialized(object): |
|
52 | 59 | |
|
53 | 60 | implements(ISerialized) |
|
54 | 61 | |
|
55 | 62 | def __init__(self, data, typeDescriptor, metadata={}): |
|
56 | 63 | self.data = data |
|
57 | 64 | self.typeDescriptor = typeDescriptor |
|
58 | 65 | self.metadata = metadata |
|
59 | 66 | |
|
60 | 67 | def getData(self): |
|
61 | 68 | return self.data |
|
62 | 69 | |
|
63 | 70 | def getDataSize(self, units=10.0**6): |
|
64 | 71 | return len(self.data)/units |
|
65 | 72 | |
|
66 | 73 | def getTypeDescriptor(self): |
|
67 | 74 | return self.typeDescriptor |
|
68 | 75 | |
|
69 | 76 | def getMetadata(self): |
|
70 | 77 | return self.metadata |
|
71 | 78 | |
|
72 | 79 | |
|
73 | 80 | class UnSerialized(object): |
|
74 | 81 | |
|
75 | 82 | implements(IUnSerialized) |
|
76 | 83 | |
|
77 | 84 | def __init__(self, obj): |
|
78 | 85 | self.obj = obj |
|
79 | 86 | |
|
80 | 87 | def getObject(self): |
|
81 | 88 | return self.obj |
|
82 | 89 | |
|
83 | 90 | |
|
84 | 91 | class SerializeIt(object): |
|
85 | 92 | |
|
86 | 93 | implements(ISerialized) |
|
87 | 94 | |
|
88 | 95 | def __init__(self, unSerialized): |
|
89 | 96 | self.data = None |
|
90 | 97 | self.obj = unSerialized.getObject() |
|
91 | 98 | if globals().has_key('numpy'): |
|
92 | 99 | if isinstance(self.obj, numpy.ndarray): |
|
93 | 100 | if len(self.obj) == 0: # length 0 arrays can't be reconstructed |
|
94 | 101 | raise SerializationError("You cannot send a length 0 array") |
|
95 | 102 | self.obj = numpy.ascontiguousarray(self.obj, dtype=None) |
|
96 | 103 | self.typeDescriptor = 'ndarray' |
|
97 | 104 | self.metadata = {'shape':self.obj.shape, |
|
98 | 105 | 'dtype':self.obj.dtype.str} |
|
99 | 106 | else: |
|
100 | 107 | self.typeDescriptor = 'pickle' |
|
101 | 108 | self.metadata = {} |
|
102 | 109 | else: |
|
103 | 110 | self.typeDescriptor = 'pickle' |
|
104 | 111 | self.metadata = {} |
|
105 | 112 | self._generateData() |
|
106 | 113 | |
|
107 | 114 | def _generateData(self): |
|
108 | 115 | if self.typeDescriptor == 'ndarray': |
|
109 | 116 | self.data = numpy.getbuffer(self.obj) |
|
110 | 117 | elif self.typeDescriptor == 'pickle': |
|
111 | 118 | self.data = pickle.dumps(self.obj, 2) |
|
112 | 119 | else: |
|
113 | 120 | raise SerializationError("Really wierd serialization error.") |
|
114 | 121 | del self.obj |
|
115 | 122 | |
|
116 | 123 | def getData(self): |
|
117 | 124 | return self.data |
|
118 | 125 | |
|
119 | 126 | def getDataSize(self, units=10.0**6): |
|
120 | 127 | return len(self.data)/units |
|
121 | 128 | |
|
122 | 129 | def getTypeDescriptor(self): |
|
123 | 130 | return self.typeDescriptor |
|
124 | 131 | |
|
125 | 132 | def getMetadata(self): |
|
126 | 133 | return self.metadata |
|
127 | 134 | |
|
128 | 135 | |
|
129 | 136 | class UnSerializeIt(UnSerialized): |
|
130 | 137 | |
|
131 | 138 | implements(IUnSerialized) |
|
132 | 139 | |
|
133 | 140 | def __init__(self, serialized): |
|
134 | 141 | self.serialized = serialized |
|
135 | 142 | |
|
136 | 143 | def getObject(self): |
|
137 | 144 | typeDescriptor = self.serialized.getTypeDescriptor() |
|
138 | 145 | if globals().has_key('numpy'): |
|
139 | 146 | if typeDescriptor == 'ndarray': |
|
140 | 147 | result = numpy.frombuffer(self.serialized.getData(), dtype = self.serialized.metadata['dtype']) |
|
141 | 148 | result.shape = self.serialized.metadata['shape'] |
|
142 | 149 | # This is a hack to make the array writable. We are working with |
|
143 | 150 | # the numpy folks to address this issue. |
|
144 | 151 | result = result.copy() |
|
145 | 152 | elif typeDescriptor == 'pickle': |
|
146 | 153 | result = pickle.loads(self.serialized.getData()) |
|
147 | 154 | else: |
|
148 | 155 | raise SerializationError("Really wierd serialization error.") |
|
149 | 156 | elif typeDescriptor == 'pickle': |
|
150 | 157 | result = pickle.loads(self.serialized.getData()) |
|
151 | 158 | else: |
|
152 | 159 | raise SerializationError("Really wierd serialization error.") |
|
153 | 160 | return result |
|
154 | 161 | |
|
155 | 162 | components.registerAdapter(UnSerializeIt, ISerialized, IUnSerialized) |
|
156 | 163 | |
|
157 | 164 | components.registerAdapter(SerializeIt, IUnSerialized, ISerialized) |
|
158 | 165 | |
|
159 | 166 | def serialize(obj): |
|
160 | 167 | return ISerialized(UnSerialized(obj)) |
|
161 | 168 | |
|
162 | 169 | def unserialize(serialized): |
|
163 | 170 | return IUnSerialized(serialized).getObject() |
@@ -1,306 +1,327 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """IPython Test Suite Runner. |
|
3 | 3 | |
|
4 | 4 | This module provides a main entry point to a user script to test IPython |
|
5 | 5 | itself from the command line. There are two ways of running this script: |
|
6 | 6 | |
|
7 | 7 | 1. With the syntax `iptest all`. This runs our entire test suite by |
|
8 | 8 | calling this script (with different arguments) or trial recursively. This |
|
9 | 9 | causes modules and package to be tested in different processes, using nose |
|
10 | 10 | or trial where appropriate. |
|
11 | 11 | 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form |
|
12 | 12 | the script simply calls nose, but with special command line flags and |
|
13 | 13 | plugins loaded. |
|
14 | 14 | |
|
15 | 15 | For now, this script requires that both nose and twisted are installed. This |
|
16 | 16 | will change in the future. |
|
17 | 17 | """ |
|
18 | 18 | |
|
19 | 19 | #----------------------------------------------------------------------------- |
|
20 | 20 | # Module imports |
|
21 | 21 | #----------------------------------------------------------------------------- |
|
22 | 22 | |
|
23 | 23 | import os |
|
24 | 24 | import os.path as path |
|
25 | 25 | import sys |
|
26 | 26 | import subprocess |
|
27 | 27 | import tempfile |
|
28 | 28 | import time |
|
29 | 29 | import warnings |
|
30 | 30 | |
|
31 | 31 | import nose.plugins.builtin |
|
32 | 32 | from nose.core import TestProgram |
|
33 | 33 | |
|
34 | 34 | from IPython.utils.platutils import find_cmd |
|
35 | 35 | from IPython.testing.plugin.ipdoctest import IPythonDoctest |
|
36 | 36 | |
|
37 | 37 | pjoin = path.join |
|
38 | 38 | |
|
39 | 39 | #----------------------------------------------------------------------------- |
|
40 | 40 | # Logic for skipping doctests |
|
41 | 41 | #----------------------------------------------------------------------------- |
|
42 | 42 | |
|
43 | 43 | def test_for(mod): |
|
44 | 44 | """Test to see if mod is importable.""" |
|
45 | 45 | try: |
|
46 | 46 | __import__(mod) |
|
47 | 47 | except ImportError: |
|
48 | 48 | return False |
|
49 | 49 | else: |
|
50 | 50 | return True |
|
51 | 51 | |
|
52 | 52 | have_curses = test_for('_curses') |
|
53 | 53 | have_wx = test_for('wx') |
|
54 | 54 | have_wx_aui = test_for('wx.aui') |
|
55 | 55 | have_zi = test_for('zope.interface') |
|
56 | 56 | have_twisted = test_for('twisted') |
|
57 | 57 | have_foolscap = test_for('foolscap') |
|
58 | 58 | have_objc = test_for('objc') |
|
59 | 59 | have_pexpect = test_for('pexpect') |
|
60 | 60 | |
|
61 | 61 | |
|
62 | 62 | def make_exclude(): |
|
63 | 63 | |
|
64 | 64 | # For the IPythonDoctest plugin, we need to exclude certain patterns that cause |
|
65 | 65 | # testing problems. We should strive to minimize the number of skipped |
|
66 | 66 | # modules, since this means untested code. As the testing machinery |
|
67 | 67 | # solidifies, this list should eventually become empty. |
|
68 | 68 | EXCLUDE = [pjoin('IPython', 'external'), |
|
69 | 69 | pjoin('IPython', 'frontend', 'process', 'winprocess.py'), |
|
70 | 70 | pjoin('IPython_doctest_plugin'), |
|
71 | 71 | pjoin('IPython', 'extensions', 'ipy_'), |
|
72 | 72 | pjoin('IPython', 'extensions', 'PhysicalQInput'), |
|
73 | 73 | pjoin('IPython', 'extensions', 'PhysicalQInteractive'), |
|
74 | 74 | pjoin('IPython', 'extensions', 'InterpreterPasteInput'), |
|
75 | 75 | pjoin('IPython', 'extensions', 'scitedirector'), |
|
76 | 76 | pjoin('IPython', 'extensions', 'numeric_formats'), |
|
77 | 77 | pjoin('IPython', 'testing', 'attic'), |
|
78 | 78 | pjoin('IPython', 'testing', 'tools'), |
|
79 | 79 | pjoin('IPython', 'testing', 'mkdoctests') |
|
80 | 80 | ] |
|
81 | 81 | |
|
82 | 82 | if not have_wx: |
|
83 | 83 | EXCLUDE.append(pjoin('IPython', 'extensions', 'igrid')) |
|
84 | 84 | EXCLUDE.append(pjoin('IPython', 'gui')) |
|
85 | 85 | EXCLUDE.append(pjoin('IPython', 'frontend', 'wx')) |
|
86 | 86 | |
|
87 | 87 | if not have_wx_aui: |
|
88 | 88 | EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython')) |
|
89 | 89 | |
|
90 | 90 | if not have_objc: |
|
91 | 91 | EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa')) |
|
92 | 92 | |
|
93 | 93 | if not have_curses: |
|
94 | 94 | EXCLUDE.append(pjoin('IPython', 'extensions', 'ibrowse')) |
|
95 | 95 | |
|
96 | 96 | if not sys.platform == 'win32': |
|
97 | 97 | EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_win32')) |
|
98 | 98 | |
|
99 | 99 | # These have to be skipped on win32 because the use echo, rm, cd, etc. |
|
100 | 100 | # See ticket https://bugs.launchpad.net/bugs/366982 |
|
101 | 101 | if sys.platform == 'win32': |
|
102 | 102 | EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip')) |
|
103 | 103 | EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample')) |
|
104 | 104 | |
|
105 | 105 | if not os.name == 'posix': |
|
106 | 106 | EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_posix')) |
|
107 | 107 | |
|
108 | 108 | if not have_pexpect: |
|
109 | 109 | EXCLUDE.append(pjoin('IPython', 'scripts', 'irunner')) |
|
110 | 110 | |
|
111 | # This is scary. We still have things in frontend and testing that | |
|
112 | # are being tested by nose that use twisted. We need to rethink | |
|
113 | # how we are isolating dependencies in testing. | |
|
114 | if not (have_twisted and have_zi and have_foolscap): | |
|
115 | EXCLUDE.append(pjoin('IPython', 'frontend', 'asyncfrontendbase')) | |
|
116 | EXCLUDE.append(pjoin('IPython', 'frontend', 'prefilterfrontend')) | |
|
117 | EXCLUDE.append(pjoin('IPython', 'frontend', 'frontendbase')) | |
|
118 | EXCLUDE.append(pjoin('IPython', 'frontend', 'linefrontendbase')) | |
|
119 | EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', | |
|
120 | 'test_linefrontend')) | |
|
121 | EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', | |
|
122 | 'test_frontendbase')) | |
|
123 | EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', | |
|
124 | 'test_prefilterfrontend')) | |
|
125 | EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', | |
|
126 | 'test_asyncfrontendbase')), | |
|
127 | EXCLUDE.append(pjoin('IPython', 'testing', 'parametric')) | |
|
128 | EXCLUDE.append(pjoin('IPython', 'testing', 'util')) | |
|
129 | EXCLUDE.append(pjoin('IPython', 'testing', 'tests', | |
|
130 | 'test_decorators_trial')) | |
|
131 | ||
|
111 | 132 | # Skip shell always because of a bug in FakeModule. |
|
112 | 133 | EXCLUDE.append(pjoin('IPython', 'core', 'shell')) |
|
113 | 134 | |
|
114 | 135 | # This is needed for the reg-exp to match on win32 in the ipdoctest plugin. |
|
115 | 136 | if sys.platform == 'win32': |
|
116 | 137 | EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE] |
|
117 | 138 | |
|
118 | 139 | return EXCLUDE |
|
119 | 140 | |
|
120 | 141 | |
|
121 | 142 | #----------------------------------------------------------------------------- |
|
122 | 143 | # Functions and classes |
|
123 | 144 | #----------------------------------------------------------------------------- |
|
124 | 145 | |
|
125 | 146 | def run_iptest(): |
|
126 | 147 | """Run the IPython test suite using nose. |
|
127 | 148 | |
|
128 | 149 | This function is called when this script is **not** called with the form |
|
129 | 150 | `iptest all`. It simply calls nose with appropriate command line flags |
|
130 | 151 | and accepts all of the standard nose arguments. |
|
131 | 152 | """ |
|
132 | 153 | |
|
133 | 154 | warnings.filterwarnings('ignore', |
|
134 | 155 | 'This will be removed soon. Use IPython.testing.util instead') |
|
135 | 156 | |
|
136 | 157 | argv = sys.argv + [ |
|
137 | 158 | # Loading ipdoctest causes problems with Twisted. |
|
138 | 159 | # I am removing this as a temporary fix to get the |
|
139 | 160 | # test suite back into working shape. Our nose |
|
140 | 161 | # plugin needs to be gone through with a fine |
|
141 | 162 | # toothed comb to find what is causing the problem. |
|
142 | 163 | '--with-ipdoctest', |
|
143 | 164 | '--ipdoctest-tests','--ipdoctest-extension=txt', |
|
144 | 165 | '--detailed-errors', |
|
145 | 166 | |
|
146 | 167 | # We add --exe because of setuptools' imbecility (it |
|
147 | 168 | # blindly does chmod +x on ALL files). Nose does the |
|
148 | 169 | # right thing and it tries to avoid executables, |
|
149 | 170 | # setuptools unfortunately forces our hand here. This |
|
150 | 171 | # has been discussed on the distutils list and the |
|
151 | 172 | # setuptools devs refuse to fix this problem! |
|
152 | 173 | '--exe', |
|
153 | 174 | ] |
|
154 | 175 | |
|
155 | 176 | # Detect if any tests were required by explicitly calling an IPython |
|
156 | 177 | # submodule or giving a specific path |
|
157 | 178 | has_tests = False |
|
158 | 179 | for arg in sys.argv: |
|
159 | 180 | if 'IPython' in arg or arg.endswith('.py') or \ |
|
160 | 181 | (':' in arg and '.py' in arg): |
|
161 | 182 | has_tests = True |
|
162 | 183 | break |
|
163 | 184 | |
|
164 | 185 | # If nothing was specifically requested, test full IPython |
|
165 | 186 | if not has_tests: |
|
166 | 187 | argv.append('IPython') |
|
167 | 188 | |
|
168 | 189 | # Construct list of plugins, omitting the existing doctest plugin, which |
|
169 | 190 | # ours replaces (and extends). |
|
170 | 191 | EXCLUDE = make_exclude() |
|
171 | 192 | plugins = [IPythonDoctest(EXCLUDE)] |
|
172 | 193 | for p in nose.plugins.builtin.plugins: |
|
173 | 194 | plug = p() |
|
174 | 195 | if plug.name == 'doctest': |
|
175 | 196 | continue |
|
176 | 197 | plugins.append(plug) |
|
177 | 198 | |
|
178 | 199 | TestProgram(argv=argv,plugins=plugins) |
|
179 | 200 | |
|
180 | 201 | |
|
181 | 202 | class IPTester(object): |
|
182 | 203 | """Call that calls iptest or trial in a subprocess. |
|
183 | 204 | """ |
|
184 | 205 | def __init__(self,runner='iptest',params=None): |
|
185 | 206 | """ """ |
|
186 | 207 | if runner == 'iptest': |
|
187 | 208 | self.runner = ['iptest','-v'] |
|
188 | 209 | else: |
|
189 | 210 | self.runner = [find_cmd('trial')] |
|
190 | 211 | if params is None: |
|
191 | 212 | params = [] |
|
192 | 213 | if isinstance(params,str): |
|
193 | 214 | params = [params] |
|
194 | 215 | self.params = params |
|
195 | 216 | |
|
196 | 217 | # Assemble call |
|
197 | 218 | self.call_args = self.runner+self.params |
|
198 | 219 | |
|
199 | 220 | if sys.platform == 'win32': |
|
200 | 221 | def run(self): |
|
201 | 222 | """Run the stored commands""" |
|
202 | 223 | # On Windows, cd to temporary directory to run tests. Otherwise, |
|
203 | 224 | # Twisted's trial may not be able to execute 'trial IPython', since |
|
204 | 225 | # it will confuse the IPython module name with the ipython |
|
205 | 226 | # execution scripts, because the windows file system isn't case |
|
206 | 227 | # sensitive. |
|
207 | 228 | # We also use os.system instead of subprocess.call, because I was |
|
208 | 229 | # having problems with subprocess and I just don't know enough |
|
209 | 230 | # about win32 to debug this reliably. Os.system may be the 'old |
|
210 | 231 | # fashioned' way to do it, but it works just fine. If someone |
|
211 | 232 | # later can clean this up that's fine, as long as the tests run |
|
212 | 233 | # reliably in win32. |
|
213 | 234 | curdir = os.getcwd() |
|
214 | 235 | os.chdir(tempfile.gettempdir()) |
|
215 | 236 | stat = os.system(' '.join(self.call_args)) |
|
216 | 237 | os.chdir(curdir) |
|
217 | 238 | return stat |
|
218 | 239 | else: |
|
219 | 240 | def run(self): |
|
220 | 241 | """Run the stored commands""" |
|
221 | 242 | return subprocess.call(self.call_args) |
|
222 | 243 | |
|
223 | 244 | |
|
224 | 245 | def make_runners(): |
|
225 | 246 | """Define the top-level packages that need to be tested. |
|
226 | 247 | """ |
|
227 | 248 | |
|
228 | 249 | nose_packages = ['config', 'core', 'extensions', |
|
229 | 250 | 'frontend', 'lib', 'quarantine', |
|
230 | 251 | 'scripts', 'testing', 'utils'] |
|
231 | 252 | trial_packages = ['kernel'] |
|
232 | 253 | |
|
233 | 254 | if have_wx: |
|
234 | 255 | nose_packages.append('gui') |
|
235 | 256 | |
|
236 | 257 | nose_packages = ['IPython.%s' % m for m in nose_packages ] |
|
237 | 258 | trial_packages = ['IPython.%s' % m for m in trial_packages ] |
|
238 | 259 | |
|
239 | 260 | # Make runners |
|
240 | 261 | runners = dict() |
|
241 | 262 | |
|
242 | 263 | nose_runners = dict(zip(nose_packages, [IPTester(params=v) for v in nose_packages])) |
|
243 | 264 | if have_zi and have_twisted and have_foolscap: |
|
244 | 265 | trial_runners = dict(zip(trial_packages, [IPTester('trial',params=v) for v in trial_packages])) |
|
245 | 266 | runners.update(nose_runners) |
|
246 | 267 | runners.update(trial_runners) |
|
247 | 268 | |
|
248 | 269 | return runners |
|
249 | 270 | |
|
250 | 271 | |
|
251 | 272 | def run_iptestall(): |
|
252 | 273 | """Run the entire IPython test suite by calling nose and trial. |
|
253 | 274 | |
|
254 | 275 | This function constructs :class:`IPTester` instances for all IPython |
|
255 | 276 | modules and package and then runs each of them. This causes the modules |
|
256 | 277 | and packages of IPython to be tested each in their own subprocess using |
|
257 | 278 | nose or twisted.trial appropriately. |
|
258 | 279 | """ |
|
259 | 280 | |
|
260 | 281 | runners = make_runners() |
|
261 | 282 | |
|
262 | 283 | # Run all test runners, tracking execution time |
|
263 | 284 | failed = {} |
|
264 | 285 | t_start = time.time() |
|
265 | 286 | for name,runner in runners.iteritems(): |
|
266 | 287 | print '*'*77 |
|
267 | 288 | print 'IPython test group:',name |
|
268 | 289 | res = runner.run() |
|
269 | 290 | if res: |
|
270 | 291 | failed[name] = res |
|
271 | 292 | t_end = time.time() |
|
272 | 293 | t_tests = t_end - t_start |
|
273 | 294 | nrunners = len(runners) |
|
274 | 295 | nfail = len(failed) |
|
275 | 296 | # summarize results |
|
276 | 297 | |
|
277 | 298 | print '*'*77 |
|
278 | 299 | print 'Ran %s test groups in %.3fs' % (nrunners, t_tests) |
|
279 | 300 | |
|
280 | 301 | if not failed: |
|
281 | 302 | print 'OK' |
|
282 | 303 | else: |
|
283 | 304 | # If anything went wrong, point out what command to rerun manually to |
|
284 | 305 | # see the actual errors and individual summary |
|
285 | 306 | print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners) |
|
286 | 307 | for name in failed: |
|
287 | 308 | failed_runner = runners[name] |
|
288 | 309 | print '-'*40 |
|
289 | 310 | print 'Runner failed:',name |
|
290 | 311 | print 'You may wish to rerun this one individually, with:' |
|
291 | 312 | print ' '.join(failed_runner.call_args) |
|
292 | 313 | |
|
293 | 314 | |
|
294 | 315 | |
|
295 | 316 | def main(): |
|
296 | 317 | if len(sys.argv) == 1: |
|
297 | 318 | run_iptestall() |
|
298 | 319 | else: |
|
299 | 320 | if sys.argv[1] == 'all': |
|
300 | 321 | run_iptestall() |
|
301 | 322 | else: |
|
302 | 323 | run_iptest() |
|
303 | 324 | |
|
304 | 325 | |
|
305 | 326 | if __name__ == '__main__': |
|
306 | 327 | main() |
General Comments 0
You need to be logged in to leave comments.
Login now