Show More
@@ -0,0 +1,41 b'' | |||
|
1 | """Minimal script to reproduce our nasty reference counting bug. | |
|
2 | ||
|
3 | The problem is related to https://bugs.launchpad.net/ipython/+bug/269966 | |
|
4 | ||
|
5 | The original fix for that appeared to work, but John D. Hunter found a | |
|
6 | matplotlib example which, when run twice in a row, would break. The problem | |
|
7 | were references held by open figures to internals of Tkinter. | |
|
8 | ||
|
9 | This code reproduces the problem that John saw, without matplotlib. | |
|
10 | ||
|
11 | This script is meant to be called by other parts of the test suite that call it | |
|
12 | via %run as if it were executed interactively by the user. As of 2009-04-13, | |
|
13 | test_magic.py calls it. | |
|
14 | """ | |
|
15 | ||
|
16 | #----------------------------------------------------------------------------- | |
|
17 | # Module imports | |
|
18 | #----------------------------------------------------------------------------- | |
|
19 | import sys | |
|
20 | ||
|
21 | from IPython import ipapi | |
|
22 | ||
|
23 | #----------------------------------------------------------------------------- | |
|
24 | # Globals | |
|
25 | #----------------------------------------------------------------------------- | |
|
26 | ip = ipapi.get() | |
|
27 | ||
|
28 | if not '_refbug_cache' in ip.user_ns: | |
|
29 | ip.user_ns['_refbug_cache'] = [] | |
|
30 | ||
|
31 | ||
|
32 | aglobal = 'Hello' | |
|
33 | def f(): | |
|
34 | return aglobal | |
|
35 | ||
|
36 | cache = ip.user_ns['_refbug_cache'] | |
|
37 | cache.append(f) | |
|
38 | ||
|
39 | def call_f(): | |
|
40 | for func in cache: | |
|
41 | print 'lowercased:',func().lower() |
@@ -0,0 +1,17 b'' | |||
|
1 | """Tests for the FakeModule objects. | |
|
2 | """ | |
|
3 | ||
|
4 | import nose.tools as nt | |
|
5 | ||
|
6 | from IPython.FakeModule import FakeModule, init_fakemod_dict | |
|
7 | ||
|
8 | # Make a fakemod and check a few properties | |
|
9 | def test_mk_fakemod(): | |
|
10 | fm = FakeModule() | |
|
11 | yield nt.assert_true,fm | |
|
12 | yield nt.assert_true,lambda : hasattr(fm,'__file__') | |
|
13 | ||
|
14 | def test_mk_fakemod_fromdict(): | |
|
15 | """Test making a FakeModule object with initial data""" | |
|
16 | fm = FakeModule(dict(hello=True)) | |
|
17 | nt.assert_true(fm.hello) |
@@ -1,42 +1,66 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """ |
|
3 | 3 | Class which mimics a module. |
|
4 | 4 | |
|
5 | 5 | Needed to allow pickle to correctly resolve namespaces during IPython |
|
6 | 6 | sessions. |
|
7 | 7 | """ |
|
8 | 8 | |
|
9 | 9 | #***************************************************************************** |
|
10 | 10 | # Copyright (C) 2002-2004 Fernando Perez. <fperez@colorado.edu> |
|
11 | 11 | # |
|
12 | 12 | # Distributed under the terms of the BSD License. The full license is in |
|
13 | 13 | # the file COPYING, distributed as part of this software. |
|
14 | 14 | #***************************************************************************** |
|
15 | 15 | |
|
16 | 16 | import types |
|
17 | 17 | |
|
18 | def init_fakemod_dict(fm,adict=None): | |
|
19 | """Initialize a FakeModule instance __dict__. | |
|
20 | ||
|
21 | Kept as a standalone function and not a method so the FakeModule API can | |
|
22 | remain basically empty. | |
|
23 | ||
|
24 | This should be considered for private IPython use, used in managing | |
|
25 | namespaces for %run. | |
|
26 | ||
|
27 | Parameters | |
|
28 | ---------- | |
|
29 | ||
|
30 | fm : FakeModule instance | |
|
31 | ||
|
32 | adict : dict, optional | |
|
33 | """ | |
|
34 | ||
|
35 | dct = {} | |
|
36 | # It seems pydoc (and perhaps others) needs any module instance to | |
|
37 | # implement a __nonzero__ method, so we add it if missing: | |
|
38 | dct.setdefault('__nonzero__',lambda : True) | |
|
39 | dct.setdefault('__file__',__file__) | |
|
40 | ||
|
41 | if adict is not None: | |
|
42 | dct.update(adict) | |
|
43 | ||
|
44 | # Hard assignment of the object's __dict__. This is nasty but deliberate. | |
|
45 | fm.__dict__.clear() | |
|
46 | fm.__dict__.update(dct) | |
|
47 | ||
|
48 | ||
|
18 | 49 | class FakeModule(types.ModuleType): |
|
19 | 50 | """Simple class with attribute access to fake a module. |
|
20 | 51 | |
|
21 | 52 | This is not meant to replace a module, but to allow inserting a fake |
|
22 | 53 | module in sys.modules so that systems which rely on run-time module |
|
23 | 54 | importing (like shelve and pickle) work correctly in interactive IPython |
|
24 | 55 | sessions. |
|
25 | 56 | |
|
26 | 57 | Do NOT use this code for anything other than this IPython private hack.""" |
|
27 | 58 | |
|
28 | 59 | def __init__(self,adict=None): |
|
29 | 60 | |
|
30 | 61 | # tmp to force __dict__ instance creation, else self.__dict__ fails |
|
31 | 62 | self.__iptmp = None |
|
32 | ||
|
33 | # It seems pydoc (and perhaps others) needs any module instance to | |
|
34 | # implement a __nonzero__ method, so we add it if missing: | |
|
35 | self.__dict__.setdefault('__nonzero__',lambda : True) | |
|
36 | self.__dict__.setdefault('__file__',__file__) | |
|
37 | ||
|
38 | 63 | # cleanup our temp trick |
|
39 | 64 | del self.__iptmp |
|
40 | ||
|
41 | if adict is not None: | |
|
42 | self.__dict__.update(adict) | |
|
65 | # Now, initialize the actual data in the instance dict. | |
|
66 | init_fakemod_dict(self,adict) |
@@ -1,3425 +1,3446 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """Magic functions for InteractiveShell. |
|
3 | 3 | """ |
|
4 | 4 | |
|
5 | 5 | #***************************************************************************** |
|
6 | 6 | # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de> and |
|
7 | 7 | # Copyright (C) 2001-2006 Fernando Perez <fperez@colorado.edu> |
|
8 | 8 | # |
|
9 | 9 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | 10 | # the file COPYING, distributed as part of this software. |
|
11 | 11 | #***************************************************************************** |
|
12 | 12 | |
|
13 | 13 | #**************************************************************************** |
|
14 | 14 | # Modules and globals |
|
15 | 15 | |
|
16 | 16 | # Python standard modules |
|
17 | 17 | import __builtin__ |
|
18 | 18 | import bdb |
|
19 | 19 | import inspect |
|
20 | 20 | import os |
|
21 | 21 | import pdb |
|
22 | 22 | import pydoc |
|
23 | 23 | import sys |
|
24 | 24 | import re |
|
25 | 25 | import tempfile |
|
26 | 26 | import time |
|
27 | 27 | import cPickle as pickle |
|
28 | 28 | import textwrap |
|
29 | 29 | from cStringIO import StringIO |
|
30 | 30 | from getopt import getopt,GetoptError |
|
31 | 31 | from pprint import pprint, pformat |
|
32 | 32 | |
|
33 | 33 | # cProfile was added in Python2.5 |
|
34 | 34 | try: |
|
35 | 35 | import cProfile as profile |
|
36 | 36 | import pstats |
|
37 | 37 | except ImportError: |
|
38 | 38 | # profile isn't bundled by default in Debian for license reasons |
|
39 | 39 | try: |
|
40 | 40 | import profile,pstats |
|
41 | 41 | except ImportError: |
|
42 | 42 | profile = pstats = None |
|
43 | 43 | |
|
44 | 44 | # Homebrewed |
|
45 | 45 | import IPython |
|
46 | 46 | from IPython import Debugger, OInspect, wildcard |
|
47 | 47 | from IPython.FakeModule import FakeModule |
|
48 | 48 | from IPython.Itpl import Itpl, itpl, printpl,itplns |
|
49 | 49 | from IPython.PyColorize import Parser |
|
50 | 50 | from IPython.ipstruct import Struct |
|
51 | 51 | from IPython.macro import Macro |
|
52 | 52 | from IPython.genutils import * |
|
53 | 53 | from IPython import platutils |
|
54 | 54 | import IPython.generics |
|
55 | 55 | import IPython.ipapi |
|
56 | 56 | from IPython.ipapi import UsageError |
|
57 | 57 | from IPython.testing import decorators as testdec |
|
58 | 58 | |
|
59 | 59 | #*************************************************************************** |
|
60 | 60 | # Utility functions |
|
61 | 61 | def on_off(tag): |
|
62 | 62 | """Return an ON/OFF string for a 1/0 input. Simple utility function.""" |
|
63 | 63 | return ['OFF','ON'][tag] |
|
64 | 64 | |
|
65 | 65 | class Bunch: pass |
|
66 | 66 | |
|
67 | 67 | def compress_dhist(dh): |
|
68 | 68 | head, tail = dh[:-10], dh[-10:] |
|
69 | 69 | |
|
70 | 70 | newhead = [] |
|
71 | 71 | done = set() |
|
72 | 72 | for h in head: |
|
73 | 73 | if h in done: |
|
74 | 74 | continue |
|
75 | 75 | newhead.append(h) |
|
76 | 76 | done.add(h) |
|
77 | 77 | |
|
78 | 78 | return newhead + tail |
|
79 | 79 | |
|
80 | 80 | |
|
81 | 81 | #*************************************************************************** |
|
82 | 82 | # Main class implementing Magic functionality |
|
83 | 83 | class Magic: |
|
84 | 84 | """Magic functions for InteractiveShell. |
|
85 | 85 | |
|
86 | 86 | Shell functions which can be reached as %function_name. All magic |
|
87 | 87 | functions should accept a string, which they can parse for their own |
|
88 | 88 | needs. This can make some functions easier to type, eg `%cd ../` |
|
89 | 89 | vs. `%cd("../")` |
|
90 | 90 | |
|
91 | 91 | ALL definitions MUST begin with the prefix magic_. The user won't need it |
|
92 | 92 | at the command line, but it is is needed in the definition. """ |
|
93 | 93 | |
|
94 | 94 | # class globals |
|
95 | 95 | auto_status = ['Automagic is OFF, % prefix IS needed for magic functions.', |
|
96 | 96 | 'Automagic is ON, % prefix NOT needed for magic functions.'] |
|
97 | 97 | |
|
98 | 98 | #...................................................................... |
|
99 | 99 | # some utility functions |
|
100 | 100 | |
|
101 | 101 | def __init__(self,shell): |
|
102 | 102 | |
|
103 | 103 | self.options_table = {} |
|
104 | 104 | if profile is None: |
|
105 | 105 | self.magic_prun = self.profile_missing_notice |
|
106 | 106 | self.shell = shell |
|
107 | 107 | |
|
108 | 108 | # namespace for holding state we may need |
|
109 | 109 | self._magic_state = Bunch() |
|
110 | 110 | |
|
111 | 111 | def profile_missing_notice(self, *args, **kwargs): |
|
112 | 112 | error("""\ |
|
113 | 113 | The profile module could not be found. It has been removed from the standard |
|
114 | 114 | python packages because of its non-free license. To use profiling, install the |
|
115 | 115 | python-profiler package from non-free.""") |
|
116 | 116 | |
|
117 | 117 | def default_option(self,fn,optstr): |
|
118 | 118 | """Make an entry in the options_table for fn, with value optstr""" |
|
119 | 119 | |
|
120 | 120 | if fn not in self.lsmagic(): |
|
121 | 121 | error("%s is not a magic function" % fn) |
|
122 | 122 | self.options_table[fn] = optstr |
|
123 | 123 | |
|
124 | 124 | def lsmagic(self): |
|
125 | 125 | """Return a list of currently available magic functions. |
|
126 | 126 | |
|
127 | 127 | Gives a list of the bare names after mangling (['ls','cd', ...], not |
|
128 | 128 | ['magic_ls','magic_cd',...]""" |
|
129 | 129 | |
|
130 | 130 | # FIXME. This needs a cleanup, in the way the magics list is built. |
|
131 | 131 | |
|
132 | 132 | # magics in class definition |
|
133 | 133 | class_magic = lambda fn: fn.startswith('magic_') and \ |
|
134 | 134 | callable(Magic.__dict__[fn]) |
|
135 | 135 | # in instance namespace (run-time user additions) |
|
136 | 136 | inst_magic = lambda fn: fn.startswith('magic_') and \ |
|
137 | 137 | callable(self.__dict__[fn]) |
|
138 | 138 | # and bound magics by user (so they can access self): |
|
139 | 139 | inst_bound_magic = lambda fn: fn.startswith('magic_') and \ |
|
140 | 140 | callable(self.__class__.__dict__[fn]) |
|
141 | 141 | magics = filter(class_magic,Magic.__dict__.keys()) + \ |
|
142 | 142 | filter(inst_magic,self.__dict__.keys()) + \ |
|
143 | 143 | filter(inst_bound_magic,self.__class__.__dict__.keys()) |
|
144 | 144 | out = [] |
|
145 | 145 | for fn in set(magics): |
|
146 | 146 | out.append(fn.replace('magic_','',1)) |
|
147 | 147 | out.sort() |
|
148 | 148 | return out |
|
149 | 149 | |
|
150 | 150 | def extract_input_slices(self,slices,raw=False): |
|
151 | 151 | """Return as a string a set of input history slices. |
|
152 | 152 | |
|
153 | 153 | Inputs: |
|
154 | 154 | |
|
155 | 155 | - slices: the set of slices is given as a list of strings (like |
|
156 | 156 | ['1','4:8','9'], since this function is for use by magic functions |
|
157 | 157 | which get their arguments as strings. |
|
158 | 158 | |
|
159 | 159 | Optional inputs: |
|
160 | 160 | |
|
161 | 161 | - raw(False): by default, the processed input is used. If this is |
|
162 | 162 | true, the raw input history is used instead. |
|
163 | 163 | |
|
164 | 164 | Note that slices can be called with two notations: |
|
165 | 165 | |
|
166 | 166 | N:M -> standard python form, means including items N...(M-1). |
|
167 | 167 | |
|
168 | 168 | N-M -> include items N..M (closed endpoint).""" |
|
169 | 169 | |
|
170 | 170 | if raw: |
|
171 | 171 | hist = self.shell.input_hist_raw |
|
172 | 172 | else: |
|
173 | 173 | hist = self.shell.input_hist |
|
174 | 174 | |
|
175 | 175 | cmds = [] |
|
176 | 176 | for chunk in slices: |
|
177 | 177 | if ':' in chunk: |
|
178 | 178 | ini,fin = map(int,chunk.split(':')) |
|
179 | 179 | elif '-' in chunk: |
|
180 | 180 | ini,fin = map(int,chunk.split('-')) |
|
181 | 181 | fin += 1 |
|
182 | 182 | else: |
|
183 | 183 | ini = int(chunk) |
|
184 | 184 | fin = ini+1 |
|
185 | 185 | cmds.append(hist[ini:fin]) |
|
186 | 186 | return cmds |
|
187 | 187 | |
|
188 | 188 | def _ofind(self, oname, namespaces=None): |
|
189 | 189 | """Find an object in the available namespaces. |
|
190 | 190 | |
|
191 | 191 | self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic |
|
192 | 192 | |
|
193 | 193 | Has special code to detect magic functions. |
|
194 | 194 | """ |
|
195 | 195 | |
|
196 | 196 | oname = oname.strip() |
|
197 | 197 | |
|
198 | 198 | alias_ns = None |
|
199 | 199 | if namespaces is None: |
|
200 | 200 | # Namespaces to search in: |
|
201 | 201 | # Put them in a list. The order is important so that we |
|
202 | 202 | # find things in the same order that Python finds them. |
|
203 | 203 | namespaces = [ ('Interactive', self.shell.user_ns), |
|
204 | 204 | ('IPython internal', self.shell.internal_ns), |
|
205 | 205 | ('Python builtin', __builtin__.__dict__), |
|
206 | 206 | ('Alias', self.shell.alias_table), |
|
207 | 207 | ] |
|
208 | 208 | alias_ns = self.shell.alias_table |
|
209 | 209 | |
|
210 | 210 | # initialize results to 'null' |
|
211 | 211 | found = 0; obj = None; ospace = None; ds = None; |
|
212 | 212 | ismagic = 0; isalias = 0; parent = None |
|
213 | 213 | |
|
214 | 214 | # Look for the given name by splitting it in parts. If the head is |
|
215 | 215 | # found, then we look for all the remaining parts as members, and only |
|
216 | 216 | # declare success if we can find them all. |
|
217 | 217 | oname_parts = oname.split('.') |
|
218 | 218 | oname_head, oname_rest = oname_parts[0],oname_parts[1:] |
|
219 | 219 | for nsname,ns in namespaces: |
|
220 | 220 | try: |
|
221 | 221 | obj = ns[oname_head] |
|
222 | 222 | except KeyError: |
|
223 | 223 | continue |
|
224 | 224 | else: |
|
225 | 225 | #print 'oname_rest:', oname_rest # dbg |
|
226 | 226 | for part in oname_rest: |
|
227 | 227 | try: |
|
228 | 228 | parent = obj |
|
229 | 229 | obj = getattr(obj,part) |
|
230 | 230 | except: |
|
231 | 231 | # Blanket except b/c some badly implemented objects |
|
232 | 232 | # allow __getattr__ to raise exceptions other than |
|
233 | 233 | # AttributeError, which then crashes IPython. |
|
234 | 234 | break |
|
235 | 235 | else: |
|
236 | 236 | # If we finish the for loop (no break), we got all members |
|
237 | 237 | found = 1 |
|
238 | 238 | ospace = nsname |
|
239 | 239 | if ns == alias_ns: |
|
240 | 240 | isalias = 1 |
|
241 | 241 | break # namespace loop |
|
242 | 242 | |
|
243 | 243 | # Try to see if it's magic |
|
244 | 244 | if not found: |
|
245 | 245 | if oname.startswith(self.shell.ESC_MAGIC): |
|
246 | 246 | oname = oname[1:] |
|
247 | 247 | obj = getattr(self,'magic_'+oname,None) |
|
248 | 248 | if obj is not None: |
|
249 | 249 | found = 1 |
|
250 | 250 | ospace = 'IPython internal' |
|
251 | 251 | ismagic = 1 |
|
252 | 252 | |
|
253 | 253 | # Last try: special-case some literals like '', [], {}, etc: |
|
254 | 254 | if not found and oname_head in ["''",'""','[]','{}','()']: |
|
255 | 255 | obj = eval(oname_head) |
|
256 | 256 | found = 1 |
|
257 | 257 | ospace = 'Interactive' |
|
258 | 258 | |
|
259 | 259 | return {'found':found, 'obj':obj, 'namespace':ospace, |
|
260 | 260 | 'ismagic':ismagic, 'isalias':isalias, 'parent':parent} |
|
261 | 261 | |
|
262 | 262 | def arg_err(self,func): |
|
263 | 263 | """Print docstring if incorrect arguments were passed""" |
|
264 | 264 | print 'Error in arguments:' |
|
265 | 265 | print OInspect.getdoc(func) |
|
266 | 266 | |
|
267 | 267 | def format_latex(self,strng): |
|
268 | 268 | """Format a string for latex inclusion.""" |
|
269 | 269 | |
|
270 | 270 | # Characters that need to be escaped for latex: |
|
271 | 271 | escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE) |
|
272 | 272 | # Magic command names as headers: |
|
273 | 273 | cmd_name_re = re.compile(r'^(%s.*?):' % self.shell.ESC_MAGIC, |
|
274 | 274 | re.MULTILINE) |
|
275 | 275 | # Magic commands |
|
276 | 276 | cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % self.shell.ESC_MAGIC, |
|
277 | 277 | re.MULTILINE) |
|
278 | 278 | # Paragraph continue |
|
279 | 279 | par_re = re.compile(r'\\$',re.MULTILINE) |
|
280 | 280 | |
|
281 | 281 | # The "\n" symbol |
|
282 | 282 | newline_re = re.compile(r'\\n') |
|
283 | 283 | |
|
284 | 284 | # Now build the string for output: |
|
285 | 285 | #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng) |
|
286 | 286 | strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:', |
|
287 | 287 | strng) |
|
288 | 288 | strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng) |
|
289 | 289 | strng = par_re.sub(r'\\\\',strng) |
|
290 | 290 | strng = escape_re.sub(r'\\\1',strng) |
|
291 | 291 | strng = newline_re.sub(r'\\textbackslash{}n',strng) |
|
292 | 292 | return strng |
|
293 | 293 | |
|
294 | 294 | def format_screen(self,strng): |
|
295 | 295 | """Format a string for screen printing. |
|
296 | 296 | |
|
297 | 297 | This removes some latex-type format codes.""" |
|
298 | 298 | # Paragraph continue |
|
299 | 299 | par_re = re.compile(r'\\$',re.MULTILINE) |
|
300 | 300 | strng = par_re.sub('',strng) |
|
301 | 301 | return strng |
|
302 | 302 | |
|
303 | 303 | def parse_options(self,arg_str,opt_str,*long_opts,**kw): |
|
304 | 304 | """Parse options passed to an argument string. |
|
305 | 305 | |
|
306 | 306 | The interface is similar to that of getopt(), but it returns back a |
|
307 | 307 | Struct with the options as keys and the stripped argument string still |
|
308 | 308 | as a string. |
|
309 | 309 | |
|
310 | 310 | arg_str is quoted as a true sys.argv vector by using shlex.split. |
|
311 | 311 | This allows us to easily expand variables, glob files, quote |
|
312 | 312 | arguments, etc. |
|
313 | 313 | |
|
314 | 314 | Options: |
|
315 | 315 | -mode: default 'string'. If given as 'list', the argument string is |
|
316 | 316 | returned as a list (split on whitespace) instead of a string. |
|
317 | 317 | |
|
318 | 318 | -list_all: put all option values in lists. Normally only options |
|
319 | 319 | appearing more than once are put in a list. |
|
320 | 320 | |
|
321 | 321 | -posix (True): whether to split the input line in POSIX mode or not, |
|
322 | 322 | as per the conventions outlined in the shlex module from the |
|
323 | 323 | standard library.""" |
|
324 | 324 | |
|
325 | 325 | # inject default options at the beginning of the input line |
|
326 | 326 | caller = sys._getframe(1).f_code.co_name.replace('magic_','') |
|
327 | 327 | arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str) |
|
328 | 328 | |
|
329 | 329 | mode = kw.get('mode','string') |
|
330 | 330 | if mode not in ['string','list']: |
|
331 | 331 | raise ValueError,'incorrect mode given: %s' % mode |
|
332 | 332 | # Get options |
|
333 | 333 | list_all = kw.get('list_all',0) |
|
334 | 334 | posix = kw.get('posix',True) |
|
335 | 335 | |
|
336 | 336 | # Check if we have more than one argument to warrant extra processing: |
|
337 | 337 | odict = {} # Dictionary with options |
|
338 | 338 | args = arg_str.split() |
|
339 | 339 | if len(args) >= 1: |
|
340 | 340 | # If the list of inputs only has 0 or 1 thing in it, there's no |
|
341 | 341 | # need to look for options |
|
342 | 342 | argv = arg_split(arg_str,posix) |
|
343 | 343 | # Do regular option processing |
|
344 | 344 | try: |
|
345 | 345 | opts,args = getopt(argv,opt_str,*long_opts) |
|
346 | 346 | except GetoptError,e: |
|
347 | 347 | raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str, |
|
348 | 348 | " ".join(long_opts))) |
|
349 | 349 | for o,a in opts: |
|
350 | 350 | if o.startswith('--'): |
|
351 | 351 | o = o[2:] |
|
352 | 352 | else: |
|
353 | 353 | o = o[1:] |
|
354 | 354 | try: |
|
355 | 355 | odict[o].append(a) |
|
356 | 356 | except AttributeError: |
|
357 | 357 | odict[o] = [odict[o],a] |
|
358 | 358 | except KeyError: |
|
359 | 359 | if list_all: |
|
360 | 360 | odict[o] = [a] |
|
361 | 361 | else: |
|
362 | 362 | odict[o] = a |
|
363 | 363 | |
|
364 | 364 | # Prepare opts,args for return |
|
365 | 365 | opts = Struct(odict) |
|
366 | 366 | if mode == 'string': |
|
367 | 367 | args = ' '.join(args) |
|
368 | 368 | |
|
369 | 369 | return opts,args |
|
370 | 370 | |
|
371 | 371 | #...................................................................... |
|
372 | 372 | # And now the actual magic functions |
|
373 | 373 | |
|
374 | 374 | # Functions for IPython shell work (vars,funcs, config, etc) |
|
375 | 375 | def magic_lsmagic(self, parameter_s = ''): |
|
376 | 376 | """List currently available magic functions.""" |
|
377 | 377 | mesc = self.shell.ESC_MAGIC |
|
378 | 378 | print 'Available magic functions:\n'+mesc+\ |
|
379 | 379 | (' '+mesc).join(self.lsmagic()) |
|
380 | 380 | print '\n' + Magic.auto_status[self.shell.rc.automagic] |
|
381 | 381 | return None |
|
382 | 382 | |
|
383 | 383 | def magic_magic(self, parameter_s = ''): |
|
384 | 384 | """Print information about the magic function system. |
|
385 | 385 | |
|
386 | 386 | Supported formats: -latex, -brief, -rest |
|
387 | 387 | """ |
|
388 | 388 | |
|
389 | 389 | mode = '' |
|
390 | 390 | try: |
|
391 | 391 | if parameter_s.split()[0] == '-latex': |
|
392 | 392 | mode = 'latex' |
|
393 | 393 | if parameter_s.split()[0] == '-brief': |
|
394 | 394 | mode = 'brief' |
|
395 | 395 | if parameter_s.split()[0] == '-rest': |
|
396 | 396 | mode = 'rest' |
|
397 | 397 | rest_docs = [] |
|
398 | 398 | except: |
|
399 | 399 | pass |
|
400 | 400 | |
|
401 | 401 | magic_docs = [] |
|
402 | 402 | for fname in self.lsmagic(): |
|
403 | 403 | mname = 'magic_' + fname |
|
404 | 404 | for space in (Magic,self,self.__class__): |
|
405 | 405 | try: |
|
406 | 406 | fn = space.__dict__[mname] |
|
407 | 407 | except KeyError: |
|
408 | 408 | pass |
|
409 | 409 | else: |
|
410 | 410 | break |
|
411 | 411 | if mode == 'brief': |
|
412 | 412 | # only first line |
|
413 | 413 | if fn.__doc__: |
|
414 | 414 | fndoc = fn.__doc__.split('\n',1)[0] |
|
415 | 415 | else: |
|
416 | 416 | fndoc = 'No documentation' |
|
417 | 417 | else: |
|
418 | 418 | if fn.__doc__: |
|
419 | 419 | fndoc = fn.__doc__.rstrip() |
|
420 | 420 | else: |
|
421 | 421 | fndoc = 'No documentation' |
|
422 | 422 | |
|
423 | 423 | |
|
424 | 424 | if mode == 'rest': |
|
425 | 425 | rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(self.shell.ESC_MAGIC, |
|
426 | 426 | fname,fndoc)) |
|
427 | 427 | |
|
428 | 428 | else: |
|
429 | 429 | magic_docs.append('%s%s:\n\t%s\n' %(self.shell.ESC_MAGIC, |
|
430 | 430 | fname,fndoc)) |
|
431 | 431 | |
|
432 | 432 | magic_docs = ''.join(magic_docs) |
|
433 | 433 | |
|
434 | 434 | if mode == 'rest': |
|
435 | 435 | return "".join(rest_docs) |
|
436 | 436 | |
|
437 | 437 | if mode == 'latex': |
|
438 | 438 | print self.format_latex(magic_docs) |
|
439 | 439 | return |
|
440 | 440 | else: |
|
441 | 441 | magic_docs = self.format_screen(magic_docs) |
|
442 | 442 | if mode == 'brief': |
|
443 | 443 | return magic_docs |
|
444 | 444 | |
|
445 | 445 | outmsg = """ |
|
446 | 446 | IPython's 'magic' functions |
|
447 | 447 | =========================== |
|
448 | 448 | |
|
449 | 449 | The magic function system provides a series of functions which allow you to |
|
450 | 450 | control the behavior of IPython itself, plus a lot of system-type |
|
451 | 451 | features. All these functions are prefixed with a % character, but parameters |
|
452 | 452 | are given without parentheses or quotes. |
|
453 | 453 | |
|
454 | 454 | NOTE: If you have 'automagic' enabled (via the command line option or with the |
|
455 | 455 | %automagic function), you don't need to type in the % explicitly. By default, |
|
456 | 456 | IPython ships with automagic on, so you should only rarely need the % escape. |
|
457 | 457 | |
|
458 | 458 | Example: typing '%cd mydir' (without the quotes) changes you working directory |
|
459 | 459 | to 'mydir', if it exists. |
|
460 | 460 | |
|
461 | 461 | You can define your own magic functions to extend the system. See the supplied |
|
462 | 462 | ipythonrc and example-magic.py files for details (in your ipython |
|
463 | 463 | configuration directory, typically $HOME/.ipython/). |
|
464 | 464 | |
|
465 | 465 | You can also define your own aliased names for magic functions. In your |
|
466 | 466 | ipythonrc file, placing a line like: |
|
467 | 467 | |
|
468 | 468 | execute __IPYTHON__.magic_pf = __IPYTHON__.magic_profile |
|
469 | 469 | |
|
470 | 470 | will define %pf as a new name for %profile. |
|
471 | 471 | |
|
472 | 472 | You can also call magics in code using the ipmagic() function, which IPython |
|
473 | 473 | automatically adds to the builtin namespace. Type 'ipmagic?' for details. |
|
474 | 474 | |
|
475 | 475 | For a list of the available magic functions, use %lsmagic. For a description |
|
476 | 476 | of any of them, type %magic_name?, e.g. '%cd?'. |
|
477 | 477 | |
|
478 | 478 | Currently the magic system has the following functions:\n""" |
|
479 | 479 | |
|
480 | 480 | mesc = self.shell.ESC_MAGIC |
|
481 | 481 | outmsg = ("%s\n%s\n\nSummary of magic functions (from %slsmagic):" |
|
482 | 482 | "\n\n%s%s\n\n%s" % (outmsg, |
|
483 | 483 | magic_docs,mesc,mesc, |
|
484 | 484 | (' '+mesc).join(self.lsmagic()), |
|
485 | 485 | Magic.auto_status[self.shell.rc.automagic] ) ) |
|
486 | 486 | |
|
487 | 487 | page(outmsg,screen_lines=self.shell.rc.screen_length) |
|
488 | 488 | |
|
489 | 489 | |
|
490 | 490 | def magic_autoindent(self, parameter_s = ''): |
|
491 | 491 | """Toggle autoindent on/off (if available).""" |
|
492 | 492 | |
|
493 | 493 | self.shell.set_autoindent() |
|
494 | 494 | print "Automatic indentation is:",['OFF','ON'][self.shell.autoindent] |
|
495 | 495 | |
|
496 | 496 | |
|
497 | 497 | def magic_automagic(self, parameter_s = ''): |
|
498 | 498 | """Make magic functions callable without having to type the initial %. |
|
499 | 499 | |
|
500 | 500 | Without argumentsl toggles on/off (when off, you must call it as |
|
501 | 501 | %automagic, of course). With arguments it sets the value, and you can |
|
502 | 502 | use any of (case insensitive): |
|
503 | 503 | |
|
504 | 504 | - on,1,True: to activate |
|
505 | 505 | |
|
506 | 506 | - off,0,False: to deactivate. |
|
507 | 507 | |
|
508 | 508 | Note that magic functions have lowest priority, so if there's a |
|
509 | 509 | variable whose name collides with that of a magic fn, automagic won't |
|
510 | 510 | work for that function (you get the variable instead). However, if you |
|
511 | 511 | delete the variable (del var), the previously shadowed magic function |
|
512 | 512 | becomes visible to automagic again.""" |
|
513 | 513 | |
|
514 | 514 | rc = self.shell.rc |
|
515 | 515 | arg = parameter_s.lower() |
|
516 | 516 | if parameter_s in ('on','1','true'): |
|
517 | 517 | rc.automagic = True |
|
518 | 518 | elif parameter_s in ('off','0','false'): |
|
519 | 519 | rc.automagic = False |
|
520 | 520 | else: |
|
521 | 521 | rc.automagic = not rc.automagic |
|
522 | 522 | print '\n' + Magic.auto_status[rc.automagic] |
|
523 | 523 | |
|
524 | 524 | @testdec.skip_doctest |
|
525 | 525 | def magic_autocall(self, parameter_s = ''): |
|
526 | 526 | """Make functions callable without having to type parentheses. |
|
527 | 527 | |
|
528 | 528 | Usage: |
|
529 | 529 | |
|
530 | 530 | %autocall [mode] |
|
531 | 531 | |
|
532 | 532 | The mode can be one of: 0->Off, 1->Smart, 2->Full. If not given, the |
|
533 | 533 | value is toggled on and off (remembering the previous state). |
|
534 | 534 | |
|
535 | 535 | In more detail, these values mean: |
|
536 | 536 | |
|
537 | 537 | 0 -> fully disabled |
|
538 | 538 | |
|
539 | 539 | 1 -> active, but do not apply if there are no arguments on the line. |
|
540 | 540 | |
|
541 | 541 | In this mode, you get: |
|
542 | 542 | |
|
543 | 543 | In [1]: callable |
|
544 | 544 | Out[1]: <built-in function callable> |
|
545 | 545 | |
|
546 | 546 | In [2]: callable 'hello' |
|
547 | 547 | ------> callable('hello') |
|
548 | 548 | Out[2]: False |
|
549 | 549 | |
|
550 | 550 | 2 -> Active always. Even if no arguments are present, the callable |
|
551 | 551 | object is called: |
|
552 | 552 | |
|
553 | 553 | In [2]: float |
|
554 | 554 | ------> float() |
|
555 | 555 | Out[2]: 0.0 |
|
556 | 556 | |
|
557 | 557 | Note that even with autocall off, you can still use '/' at the start of |
|
558 | 558 | a line to treat the first argument on the command line as a function |
|
559 | 559 | and add parentheses to it: |
|
560 | 560 | |
|
561 | 561 | In [8]: /str 43 |
|
562 | 562 | ------> str(43) |
|
563 | 563 | Out[8]: '43' |
|
564 | 564 | |
|
565 | 565 | # all-random (note for auto-testing) |
|
566 | 566 | """ |
|
567 | 567 | |
|
568 | 568 | rc = self.shell.rc |
|
569 | 569 | |
|
570 | 570 | if parameter_s: |
|
571 | 571 | arg = int(parameter_s) |
|
572 | 572 | else: |
|
573 | 573 | arg = 'toggle' |
|
574 | 574 | |
|
575 | 575 | if not arg in (0,1,2,'toggle'): |
|
576 | 576 | error('Valid modes: (0->Off, 1->Smart, 2->Full') |
|
577 | 577 | return |
|
578 | 578 | |
|
579 | 579 | if arg in (0,1,2): |
|
580 | 580 | rc.autocall = arg |
|
581 | 581 | else: # toggle |
|
582 | 582 | if rc.autocall: |
|
583 | 583 | self._magic_state.autocall_save = rc.autocall |
|
584 | 584 | rc.autocall = 0 |
|
585 | 585 | else: |
|
586 | 586 | try: |
|
587 | 587 | rc.autocall = self._magic_state.autocall_save |
|
588 | 588 | except AttributeError: |
|
589 | 589 | rc.autocall = self._magic_state.autocall_save = 1 |
|
590 | 590 | |
|
591 | 591 | print "Automatic calling is:",['OFF','Smart','Full'][rc.autocall] |
|
592 | 592 | |
|
593 | 593 | def magic_system_verbose(self, parameter_s = ''): |
|
594 | 594 | """Set verbose printing of system calls. |
|
595 | 595 | |
|
596 | 596 | If called without an argument, act as a toggle""" |
|
597 | 597 | |
|
598 | 598 | if parameter_s: |
|
599 | 599 | val = bool(eval(parameter_s)) |
|
600 | 600 | else: |
|
601 | 601 | val = None |
|
602 | 602 | |
|
603 | 603 | self.shell.rc_set_toggle('system_verbose',val) |
|
604 | 604 | print "System verbose printing is:",\ |
|
605 | 605 | ['OFF','ON'][self.shell.rc.system_verbose] |
|
606 | 606 | |
|
607 | 607 | |
|
608 | 608 | def magic_page(self, parameter_s=''): |
|
609 | 609 | """Pretty print the object and display it through a pager. |
|
610 | 610 | |
|
611 | 611 | %page [options] OBJECT |
|
612 | 612 | |
|
613 | 613 | If no object is given, use _ (last output). |
|
614 | 614 | |
|
615 | 615 | Options: |
|
616 | 616 | |
|
617 | 617 | -r: page str(object), don't pretty-print it.""" |
|
618 | 618 | |
|
619 | 619 | # After a function contributed by Olivier Aubert, slightly modified. |
|
620 | 620 | |
|
621 | 621 | # Process options/args |
|
622 | 622 | opts,args = self.parse_options(parameter_s,'r') |
|
623 | 623 | raw = 'r' in opts |
|
624 | 624 | |
|
625 | 625 | oname = args and args or '_' |
|
626 | 626 | info = self._ofind(oname) |
|
627 | 627 | if info['found']: |
|
628 | 628 | txt = (raw and str or pformat)( info['obj'] ) |
|
629 | 629 | page(txt) |
|
630 | 630 | else: |
|
631 | 631 | print 'Object `%s` not found' % oname |
|
632 | 632 | |
|
633 | 633 | def magic_profile(self, parameter_s=''): |
|
634 | 634 | """Print your currently active IPyhton profile.""" |
|
635 | 635 | if self.shell.rc.profile: |
|
636 | 636 | printpl('Current IPython profile: $self.shell.rc.profile.') |
|
637 | 637 | else: |
|
638 | 638 | print 'No profile active.' |
|
639 | 639 | |
|
640 | 640 | def magic_pinfo(self, parameter_s='', namespaces=None): |
|
641 | 641 | """Provide detailed information about an object. |
|
642 | 642 | |
|
643 | 643 | '%pinfo object' is just a synonym for object? or ?object.""" |
|
644 | 644 | |
|
645 | 645 | #print 'pinfo par: <%s>' % parameter_s # dbg |
|
646 | 646 | |
|
647 | 647 | |
|
648 | 648 | # detail_level: 0 -> obj? , 1 -> obj?? |
|
649 | 649 | detail_level = 0 |
|
650 | 650 | # We need to detect if we got called as 'pinfo pinfo foo', which can |
|
651 | 651 | # happen if the user types 'pinfo foo?' at the cmd line. |
|
652 | 652 | pinfo,qmark1,oname,qmark2 = \ |
|
653 | 653 | re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups() |
|
654 | 654 | if pinfo or qmark1 or qmark2: |
|
655 | 655 | detail_level = 1 |
|
656 | 656 | if "*" in oname: |
|
657 | 657 | self.magic_psearch(oname) |
|
658 | 658 | else: |
|
659 | 659 | self._inspect('pinfo', oname, detail_level=detail_level, |
|
660 | 660 | namespaces=namespaces) |
|
661 | 661 | |
|
662 | 662 | def magic_pdef(self, parameter_s='', namespaces=None): |
|
663 | 663 | """Print the definition header for any callable object. |
|
664 | 664 | |
|
665 | 665 | If the object is a class, print the constructor information.""" |
|
666 | 666 | self._inspect('pdef',parameter_s, namespaces) |
|
667 | 667 | |
|
668 | 668 | def magic_pdoc(self, parameter_s='', namespaces=None): |
|
669 | 669 | """Print the docstring for an object. |
|
670 | 670 | |
|
671 | 671 | If the given object is a class, it will print both the class and the |
|
672 | 672 | constructor docstrings.""" |
|
673 | 673 | self._inspect('pdoc',parameter_s, namespaces) |
|
674 | 674 | |
|
675 | 675 | def magic_psource(self, parameter_s='', namespaces=None): |
|
676 | 676 | """Print (or run through pager) the source code for an object.""" |
|
677 | 677 | self._inspect('psource',parameter_s, namespaces) |
|
678 | 678 | |
|
679 | 679 | def magic_pfile(self, parameter_s=''): |
|
680 | 680 | """Print (or run through pager) the file where an object is defined. |
|
681 | 681 | |
|
682 | 682 | The file opens at the line where the object definition begins. IPython |
|
683 | 683 | will honor the environment variable PAGER if set, and otherwise will |
|
684 | 684 | do its best to print the file in a convenient form. |
|
685 | 685 | |
|
686 | 686 | If the given argument is not an object currently defined, IPython will |
|
687 | 687 | try to interpret it as a filename (automatically adding a .py extension |
|
688 | 688 | if needed). You can thus use %pfile as a syntax highlighting code |
|
689 | 689 | viewer.""" |
|
690 | 690 | |
|
691 | 691 | # first interpret argument as an object name |
|
692 | 692 | out = self._inspect('pfile',parameter_s) |
|
693 | 693 | # if not, try the input as a filename |
|
694 | 694 | if out == 'not found': |
|
695 | 695 | try: |
|
696 | 696 | filename = get_py_filename(parameter_s) |
|
697 | 697 | except IOError,msg: |
|
698 | 698 | print msg |
|
699 | 699 | return |
|
700 | 700 | page(self.shell.inspector.format(file(filename).read())) |
|
701 | 701 | |
|
702 | 702 | def _inspect(self,meth,oname,namespaces=None,**kw): |
|
703 | 703 | """Generic interface to the inspector system. |
|
704 | 704 | |
|
705 | 705 | This function is meant to be called by pdef, pdoc & friends.""" |
|
706 | 706 | |
|
707 | 707 | #oname = oname.strip() |
|
708 | 708 | #print '1- oname: <%r>' % oname # dbg |
|
709 | 709 | try: |
|
710 | 710 | oname = oname.strip().encode('ascii') |
|
711 | 711 | #print '2- oname: <%r>' % oname # dbg |
|
712 | 712 | except UnicodeEncodeError: |
|
713 | 713 | print 'Python identifiers can only contain ascii characters.' |
|
714 | 714 | return 'not found' |
|
715 | 715 | |
|
716 | 716 | info = Struct(self._ofind(oname, namespaces)) |
|
717 | 717 | |
|
718 | 718 | if info.found: |
|
719 | 719 | try: |
|
720 | 720 | IPython.generics.inspect_object(info.obj) |
|
721 | 721 | return |
|
722 | 722 | except IPython.ipapi.TryNext: |
|
723 | 723 | pass |
|
724 | 724 | # Get the docstring of the class property if it exists. |
|
725 | 725 | path = oname.split('.') |
|
726 | 726 | root = '.'.join(path[:-1]) |
|
727 | 727 | if info.parent is not None: |
|
728 | 728 | try: |
|
729 | 729 | target = getattr(info.parent, '__class__') |
|
730 | 730 | # The object belongs to a class instance. |
|
731 | 731 | try: |
|
732 | 732 | target = getattr(target, path[-1]) |
|
733 | 733 | # The class defines the object. |
|
734 | 734 | if isinstance(target, property): |
|
735 | 735 | oname = root + '.__class__.' + path[-1] |
|
736 | 736 | info = Struct(self._ofind(oname)) |
|
737 | 737 | except AttributeError: pass |
|
738 | 738 | except AttributeError: pass |
|
739 | 739 | |
|
740 | 740 | pmethod = getattr(self.shell.inspector,meth) |
|
741 | 741 | formatter = info.ismagic and self.format_screen or None |
|
742 | 742 | if meth == 'pdoc': |
|
743 | 743 | pmethod(info.obj,oname,formatter) |
|
744 | 744 | elif meth == 'pinfo': |
|
745 | 745 | pmethod(info.obj,oname,formatter,info,**kw) |
|
746 | 746 | else: |
|
747 | 747 | pmethod(info.obj,oname) |
|
748 | 748 | else: |
|
749 | 749 | print 'Object `%s` not found.' % oname |
|
750 | 750 | return 'not found' # so callers can take other action |
|
751 | 751 | |
|
752 | 752 | def magic_psearch(self, parameter_s=''): |
|
753 | 753 | """Search for object in namespaces by wildcard. |
|
754 | 754 | |
|
755 | 755 | %psearch [options] PATTERN [OBJECT TYPE] |
|
756 | 756 | |
|
757 | 757 | Note: ? can be used as a synonym for %psearch, at the beginning or at |
|
758 | 758 | the end: both a*? and ?a* are equivalent to '%psearch a*'. Still, the |
|
759 | 759 | rest of the command line must be unchanged (options come first), so |
|
760 | 760 | for example the following forms are equivalent |
|
761 | 761 | |
|
762 | 762 | %psearch -i a* function |
|
763 | 763 | -i a* function? |
|
764 | 764 | ?-i a* function |
|
765 | 765 | |
|
766 | 766 | Arguments: |
|
767 | 767 | |
|
768 | 768 | PATTERN |
|
769 | 769 | |
|
770 | 770 | where PATTERN is a string containing * as a wildcard similar to its |
|
771 | 771 | use in a shell. The pattern is matched in all namespaces on the |
|
772 | 772 | search path. By default objects starting with a single _ are not |
|
773 | 773 | matched, many IPython generated objects have a single |
|
774 | 774 | underscore. The default is case insensitive matching. Matching is |
|
775 | 775 | also done on the attributes of objects and not only on the objects |
|
776 | 776 | in a module. |
|
777 | 777 | |
|
778 | 778 | [OBJECT TYPE] |
|
779 | 779 | |
|
780 | 780 | Is the name of a python type from the types module. The name is |
|
781 | 781 | given in lowercase without the ending type, ex. StringType is |
|
782 | 782 | written string. By adding a type here only objects matching the |
|
783 | 783 | given type are matched. Using all here makes the pattern match all |
|
784 | 784 | types (this is the default). |
|
785 | 785 | |
|
786 | 786 | Options: |
|
787 | 787 | |
|
788 | 788 | -a: makes the pattern match even objects whose names start with a |
|
789 | 789 | single underscore. These names are normally ommitted from the |
|
790 | 790 | search. |
|
791 | 791 | |
|
792 | 792 | -i/-c: make the pattern case insensitive/sensitive. If neither of |
|
793 | 793 | these options is given, the default is read from your ipythonrc |
|
794 | 794 | file. The option name which sets this value is |
|
795 | 795 | 'wildcards_case_sensitive'. If this option is not specified in your |
|
796 | 796 | ipythonrc file, IPython's internal default is to do a case sensitive |
|
797 | 797 | search. |
|
798 | 798 | |
|
799 | 799 | -e/-s NAMESPACE: exclude/search a given namespace. The pattern you |
|
800 | 800 | specifiy can be searched in any of the following namespaces: |
|
801 | 801 | 'builtin', 'user', 'user_global','internal', 'alias', where |
|
802 | 802 | 'builtin' and 'user' are the search defaults. Note that you should |
|
803 | 803 | not use quotes when specifying namespaces. |
|
804 | 804 | |
|
805 | 805 | 'Builtin' contains the python module builtin, 'user' contains all |
|
806 | 806 | user data, 'alias' only contain the shell aliases and no python |
|
807 | 807 | objects, 'internal' contains objects used by IPython. The |
|
808 | 808 | 'user_global' namespace is only used by embedded IPython instances, |
|
809 | 809 | and it contains module-level globals. You can add namespaces to the |
|
810 | 810 | search with -s or exclude them with -e (these options can be given |
|
811 | 811 | more than once). |
|
812 | 812 | |
|
813 | 813 | Examples: |
|
814 | 814 | |
|
815 | 815 | %psearch a* -> objects beginning with an a |
|
816 | 816 | %psearch -e builtin a* -> objects NOT in the builtin space starting in a |
|
817 | 817 | %psearch a* function -> all functions beginning with an a |
|
818 | 818 | %psearch re.e* -> objects beginning with an e in module re |
|
819 | 819 | %psearch r*.e* -> objects that start with e in modules starting in r |
|
820 | 820 | %psearch r*.* string -> all strings in modules beginning with r |
|
821 | 821 | |
|
822 | 822 | Case sensitve search: |
|
823 | 823 | |
|
824 | 824 | %psearch -c a* list all object beginning with lower case a |
|
825 | 825 | |
|
826 | 826 | Show objects beginning with a single _: |
|
827 | 827 | |
|
828 | 828 | %psearch -a _* list objects beginning with a single underscore""" |
|
829 | 829 | try: |
|
830 | 830 | parameter_s = parameter_s.encode('ascii') |
|
831 | 831 | except UnicodeEncodeError: |
|
832 | 832 | print 'Python identifiers can only contain ascii characters.' |
|
833 | 833 | return |
|
834 | 834 | |
|
835 | 835 | # default namespaces to be searched |
|
836 | 836 | def_search = ['user','builtin'] |
|
837 | 837 | |
|
838 | 838 | # Process options/args |
|
839 | 839 | opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True) |
|
840 | 840 | opt = opts.get |
|
841 | 841 | shell = self.shell |
|
842 | 842 | psearch = shell.inspector.psearch |
|
843 | 843 | |
|
844 | 844 | # select case options |
|
845 | 845 | if opts.has_key('i'): |
|
846 | 846 | ignore_case = True |
|
847 | 847 | elif opts.has_key('c'): |
|
848 | 848 | ignore_case = False |
|
849 | 849 | else: |
|
850 | 850 | ignore_case = not shell.rc.wildcards_case_sensitive |
|
851 | 851 | |
|
852 | 852 | # Build list of namespaces to search from user options |
|
853 | 853 | def_search.extend(opt('s',[])) |
|
854 | 854 | ns_exclude = ns_exclude=opt('e',[]) |
|
855 | 855 | ns_search = [nm for nm in def_search if nm not in ns_exclude] |
|
856 | 856 | |
|
857 | 857 | # Call the actual search |
|
858 | 858 | try: |
|
859 | 859 | psearch(args,shell.ns_table,ns_search, |
|
860 | 860 | show_all=opt('a'),ignore_case=ignore_case) |
|
861 | 861 | except: |
|
862 | 862 | shell.showtraceback() |
|
863 | 863 | |
|
864 | 864 | def magic_who_ls(self, parameter_s=''): |
|
865 | 865 | """Return a sorted list of all interactive variables. |
|
866 | 866 | |
|
867 | 867 | If arguments are given, only variables of types matching these |
|
868 | 868 | arguments are returned.""" |
|
869 | 869 | |
|
870 | 870 | user_ns = self.shell.user_ns |
|
871 | 871 | internal_ns = self.shell.internal_ns |
|
872 | 872 | user_config_ns = self.shell.user_config_ns |
|
873 | 873 | out = [] |
|
874 | 874 | typelist = parameter_s.split() |
|
875 | 875 | |
|
876 | 876 | for i in user_ns: |
|
877 | 877 | if not (i.startswith('_') or i.startswith('_i')) \ |
|
878 | 878 | and not (i in internal_ns or i in user_config_ns): |
|
879 | 879 | if typelist: |
|
880 | 880 | if type(user_ns[i]).__name__ in typelist: |
|
881 | 881 | out.append(i) |
|
882 | 882 | else: |
|
883 | 883 | out.append(i) |
|
884 | 884 | out.sort() |
|
885 | 885 | return out |
|
886 | 886 | |
|
887 | 887 | def magic_who(self, parameter_s=''): |
|
888 | 888 | """Print all interactive variables, with some minimal formatting. |
|
889 | 889 | |
|
890 | 890 | If any arguments are given, only variables whose type matches one of |
|
891 | 891 | these are printed. For example: |
|
892 | 892 | |
|
893 | 893 | %who function str |
|
894 | 894 | |
|
895 | 895 | will only list functions and strings, excluding all other types of |
|
896 | 896 | variables. To find the proper type names, simply use type(var) at a |
|
897 | 897 | command line to see how python prints type names. For example: |
|
898 | 898 | |
|
899 | 899 | In [1]: type('hello')\\ |
|
900 | 900 | Out[1]: <type 'str'> |
|
901 | 901 | |
|
902 | 902 | indicates that the type name for strings is 'str'. |
|
903 | 903 | |
|
904 | 904 | %who always excludes executed names loaded through your configuration |
|
905 | 905 | file and things which are internal to IPython. |
|
906 | 906 | |
|
907 | 907 | This is deliberate, as typically you may load many modules and the |
|
908 | 908 | purpose of %who is to show you only what you've manually defined.""" |
|
909 | 909 | |
|
910 | 910 | varlist = self.magic_who_ls(parameter_s) |
|
911 | 911 | if not varlist: |
|
912 | 912 | if parameter_s: |
|
913 | 913 | print 'No variables match your requested type.' |
|
914 | 914 | else: |
|
915 | 915 | print 'Interactive namespace is empty.' |
|
916 | 916 | return |
|
917 | 917 | |
|
918 | 918 | # if we have variables, move on... |
|
919 | 919 | count = 0 |
|
920 | 920 | for i in varlist: |
|
921 | 921 | print i+'\t', |
|
922 | 922 | count += 1 |
|
923 | 923 | if count > 8: |
|
924 | 924 | count = 0 |
|
925 | 925 | |
|
926 | 926 | |
|
927 | 927 | |
|
928 | 928 | def magic_whos(self, parameter_s=''): |
|
929 | 929 | """Like %who, but gives some extra information about each variable. |
|
930 | 930 | |
|
931 | 931 | The same type filtering of %who can be applied here. |
|
932 | 932 | |
|
933 | 933 | For all variables, the type is printed. Additionally it prints: |
|
934 | 934 | |
|
935 | 935 | - For {},[],(): their length. |
|
936 | 936 | |
|
937 | 937 | - For numpy and Numeric arrays, a summary with shape, number of |
|
938 | 938 | elements, typecode and size in memory. |
|
939 | 939 | |
|
940 | 940 | - Everything else: a string representation, snipping their middle if |
|
941 | 941 | too long.""" |
|
942 | 942 | |
|
943 | 943 | varnames = self.magic_who_ls(parameter_s) |
|
944 | 944 | if not varnames: |
|
945 | 945 | if parameter_s: |
|
946 | 946 | print 'No variables match your requested type.' |
|
947 | 947 | else: |
|
948 | 948 | print 'Interactive namespace is empty.' |
|
949 | 949 | return |
|
950 | 950 | |
|
951 | 951 | # if we have variables, move on... |
|
952 | 952 | |
|
953 | 953 | # for these types, show len() instead of data: |
|
954 | 954 | seq_types = [types.DictType,types.ListType,types.TupleType] |
|
955 | 955 | |
|
956 | 956 | # for numpy/Numeric arrays, display summary info |
|
957 | 957 | try: |
|
958 | 958 | import numpy |
|
959 | 959 | except ImportError: |
|
960 | 960 | ndarray_type = None |
|
961 | 961 | else: |
|
962 | 962 | ndarray_type = numpy.ndarray.__name__ |
|
963 | 963 | try: |
|
964 | 964 | import Numeric |
|
965 | 965 | except ImportError: |
|
966 | 966 | array_type = None |
|
967 | 967 | else: |
|
968 | 968 | array_type = Numeric.ArrayType.__name__ |
|
969 | 969 | |
|
970 | 970 | # Find all variable names and types so we can figure out column sizes |
|
971 | 971 | def get_vars(i): |
|
972 | 972 | return self.shell.user_ns[i] |
|
973 | 973 | |
|
974 | 974 | # some types are well known and can be shorter |
|
975 | 975 | abbrevs = {'IPython.macro.Macro' : 'Macro'} |
|
976 | 976 | def type_name(v): |
|
977 | 977 | tn = type(v).__name__ |
|
978 | 978 | return abbrevs.get(tn,tn) |
|
979 | 979 | |
|
980 | 980 | varlist = map(get_vars,varnames) |
|
981 | 981 | |
|
982 | 982 | typelist = [] |
|
983 | 983 | for vv in varlist: |
|
984 | 984 | tt = type_name(vv) |
|
985 | 985 | |
|
986 | 986 | if tt=='instance': |
|
987 | 987 | typelist.append( abbrevs.get(str(vv.__class__), |
|
988 | 988 | str(vv.__class__))) |
|
989 | 989 | else: |
|
990 | 990 | typelist.append(tt) |
|
991 | 991 | |
|
992 | 992 | # column labels and # of spaces as separator |
|
993 | 993 | varlabel = 'Variable' |
|
994 | 994 | typelabel = 'Type' |
|
995 | 995 | datalabel = 'Data/Info' |
|
996 | 996 | colsep = 3 |
|
997 | 997 | # variable format strings |
|
998 | 998 | vformat = "$vname.ljust(varwidth)$vtype.ljust(typewidth)" |
|
999 | 999 | vfmt_short = '$vstr[:25]<...>$vstr[-25:]' |
|
1000 | 1000 | aformat = "%s: %s elems, type `%s`, %s bytes" |
|
1001 | 1001 | # find the size of the columns to format the output nicely |
|
1002 | 1002 | varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep |
|
1003 | 1003 | typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep |
|
1004 | 1004 | # table header |
|
1005 | 1005 | print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \ |
|
1006 | 1006 | ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1) |
|
1007 | 1007 | # and the table itself |
|
1008 | 1008 | kb = 1024 |
|
1009 | 1009 | Mb = 1048576 # kb**2 |
|
1010 | 1010 | for vname,var,vtype in zip(varnames,varlist,typelist): |
|
1011 | 1011 | print itpl(vformat), |
|
1012 | 1012 | if vtype in seq_types: |
|
1013 | 1013 | print len(var) |
|
1014 | 1014 | elif vtype in [array_type,ndarray_type]: |
|
1015 | 1015 | vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1] |
|
1016 | 1016 | if vtype==ndarray_type: |
|
1017 | 1017 | # numpy |
|
1018 | 1018 | vsize = var.size |
|
1019 | 1019 | vbytes = vsize*var.itemsize |
|
1020 | 1020 | vdtype = var.dtype |
|
1021 | 1021 | else: |
|
1022 | 1022 | # Numeric |
|
1023 | 1023 | vsize = Numeric.size(var) |
|
1024 | 1024 | vbytes = vsize*var.itemsize() |
|
1025 | 1025 | vdtype = var.typecode() |
|
1026 | 1026 | |
|
1027 | 1027 | if vbytes < 100000: |
|
1028 | 1028 | print aformat % (vshape,vsize,vdtype,vbytes) |
|
1029 | 1029 | else: |
|
1030 | 1030 | print aformat % (vshape,vsize,vdtype,vbytes), |
|
1031 | 1031 | if vbytes < Mb: |
|
1032 | 1032 | print '(%s kb)' % (vbytes/kb,) |
|
1033 | 1033 | else: |
|
1034 | 1034 | print '(%s Mb)' % (vbytes/Mb,) |
|
1035 | 1035 | else: |
|
1036 | 1036 | try: |
|
1037 | 1037 | vstr = str(var) |
|
1038 | 1038 | except UnicodeEncodeError: |
|
1039 | 1039 | vstr = unicode(var).encode(sys.getdefaultencoding(), |
|
1040 | 1040 | 'backslashreplace') |
|
1041 | 1041 | vstr = vstr.replace('\n','\\n') |
|
1042 | 1042 | if len(vstr) < 50: |
|
1043 | 1043 | print vstr |
|
1044 | 1044 | else: |
|
1045 | 1045 | printpl(vfmt_short) |
|
1046 | 1046 | |
|
1047 | 1047 | def magic_reset(self, parameter_s=''): |
|
1048 | 1048 | """Resets the namespace by removing all names defined by the user. |
|
1049 | 1049 | |
|
1050 | 1050 | Input/Output history are left around in case you need them. |
|
1051 | 1051 | |
|
1052 | 1052 | Parameters |
|
1053 | 1053 | ---------- |
|
1054 | 1054 | -y : force reset without asking for confirmation. |
|
1055 | 1055 | |
|
1056 | 1056 | Examples |
|
1057 | 1057 | -------- |
|
1058 | 1058 | In [6]: a = 1 |
|
1059 | 1059 | |
|
1060 | 1060 | In [7]: a |
|
1061 | 1061 | Out[7]: 1 |
|
1062 | 1062 | |
|
1063 | 1063 | In [8]: 'a' in _ip.user_ns |
|
1064 | 1064 | Out[8]: True |
|
1065 | 1065 | |
|
1066 | 1066 | In [9]: %reset -f |
|
1067 | 1067 | |
|
1068 | 1068 | In [10]: 'a' in _ip.user_ns |
|
1069 | 1069 | Out[10]: False |
|
1070 | 1070 | """ |
|
1071 | 1071 | |
|
1072 | 1072 | if parameter_s == '-f': |
|
1073 | 1073 | ans = True |
|
1074 | 1074 | else: |
|
1075 | 1075 | ans = self.shell.ask_yes_no( |
|
1076 | 1076 | "Once deleted, variables cannot be recovered. Proceed (y/[n])? ") |
|
1077 | 1077 | if not ans: |
|
1078 | 1078 | print 'Nothing done.' |
|
1079 | 1079 | return |
|
1080 | 1080 | user_ns = self.shell.user_ns |
|
1081 | 1081 | for i in self.magic_who_ls(): |
|
1082 | 1082 | del(user_ns[i]) |
|
1083 | 1083 | |
|
1084 | 1084 | # Also flush the private list of module references kept for script |
|
1085 | 1085 | # execution protection |
|
1086 | 1086 | self.shell.clear_main_mod_cache() |
|
1087 | 1087 | |
|
1088 | 1088 | def magic_logstart(self,parameter_s=''): |
|
1089 | 1089 | """Start logging anywhere in a session. |
|
1090 | 1090 | |
|
1091 | 1091 | %logstart [-o|-r|-t] [log_name [log_mode]] |
|
1092 | 1092 | |
|
1093 | 1093 | If no name is given, it defaults to a file named 'ipython_log.py' in your |
|
1094 | 1094 | current directory, in 'rotate' mode (see below). |
|
1095 | 1095 | |
|
1096 | 1096 | '%logstart name' saves to file 'name' in 'backup' mode. It saves your |
|
1097 | 1097 | history up to that point and then continues logging. |
|
1098 | 1098 | |
|
1099 | 1099 | %logstart takes a second optional parameter: logging mode. This can be one |
|
1100 | 1100 | of (note that the modes are given unquoted):\\ |
|
1101 | 1101 | append: well, that says it.\\ |
|
1102 | 1102 | backup: rename (if exists) to name~ and start name.\\ |
|
1103 | 1103 | global: single logfile in your home dir, appended to.\\ |
|
1104 | 1104 | over : overwrite existing log.\\ |
|
1105 | 1105 | rotate: create rotating logs name.1~, name.2~, etc. |
|
1106 | 1106 | |
|
1107 | 1107 | Options: |
|
1108 | 1108 | |
|
1109 | 1109 | -o: log also IPython's output. In this mode, all commands which |
|
1110 | 1110 | generate an Out[NN] prompt are recorded to the logfile, right after |
|
1111 | 1111 | their corresponding input line. The output lines are always |
|
1112 | 1112 | prepended with a '#[Out]# ' marker, so that the log remains valid |
|
1113 | 1113 | Python code. |
|
1114 | 1114 | |
|
1115 | 1115 | Since this marker is always the same, filtering only the output from |
|
1116 | 1116 | a log is very easy, using for example a simple awk call: |
|
1117 | 1117 | |
|
1118 | 1118 | awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py |
|
1119 | 1119 | |
|
1120 | 1120 | -r: log 'raw' input. Normally, IPython's logs contain the processed |
|
1121 | 1121 | input, so that user lines are logged in their final form, converted |
|
1122 | 1122 | into valid Python. For example, %Exit is logged as |
|
1123 | 1123 | '_ip.magic("Exit"). If the -r flag is given, all input is logged |
|
1124 | 1124 | exactly as typed, with no transformations applied. |
|
1125 | 1125 | |
|
1126 | 1126 | -t: put timestamps before each input line logged (these are put in |
|
1127 | 1127 | comments).""" |
|
1128 | 1128 | |
|
1129 | 1129 | opts,par = self.parse_options(parameter_s,'ort') |
|
1130 | 1130 | log_output = 'o' in opts |
|
1131 | 1131 | log_raw_input = 'r' in opts |
|
1132 | 1132 | timestamp = 't' in opts |
|
1133 | 1133 | |
|
1134 | 1134 | rc = self.shell.rc |
|
1135 | 1135 | logger = self.shell.logger |
|
1136 | 1136 | |
|
1137 | 1137 | # if no args are given, the defaults set in the logger constructor by |
|
1138 | 1138 | # ipytohn remain valid |
|
1139 | 1139 | if par: |
|
1140 | 1140 | try: |
|
1141 | 1141 | logfname,logmode = par.split() |
|
1142 | 1142 | except: |
|
1143 | 1143 | logfname = par |
|
1144 | 1144 | logmode = 'backup' |
|
1145 | 1145 | else: |
|
1146 | 1146 | logfname = logger.logfname |
|
1147 | 1147 | logmode = logger.logmode |
|
1148 | 1148 | # put logfname into rc struct as if it had been called on the command |
|
1149 | 1149 | # line, so it ends up saved in the log header Save it in case we need |
|
1150 | 1150 | # to restore it... |
|
1151 | 1151 | old_logfile = rc.opts.get('logfile','') |
|
1152 | 1152 | if logfname: |
|
1153 | 1153 | logfname = os.path.expanduser(logfname) |
|
1154 | 1154 | rc.opts.logfile = logfname |
|
1155 | 1155 | loghead = self.shell.loghead_tpl % (rc.opts,rc.args) |
|
1156 | 1156 | try: |
|
1157 | 1157 | started = logger.logstart(logfname,loghead,logmode, |
|
1158 | 1158 | log_output,timestamp,log_raw_input) |
|
1159 | 1159 | except: |
|
1160 | 1160 | rc.opts.logfile = old_logfile |
|
1161 | 1161 | warn("Couldn't start log: %s" % sys.exc_info()[1]) |
|
1162 | 1162 | else: |
|
1163 | 1163 | # log input history up to this point, optionally interleaving |
|
1164 | 1164 | # output if requested |
|
1165 | 1165 | |
|
1166 | 1166 | if timestamp: |
|
1167 | 1167 | # disable timestamping for the previous history, since we've |
|
1168 | 1168 | # lost those already (no time machine here). |
|
1169 | 1169 | logger.timestamp = False |
|
1170 | 1170 | |
|
1171 | 1171 | if log_raw_input: |
|
1172 | 1172 | input_hist = self.shell.input_hist_raw |
|
1173 | 1173 | else: |
|
1174 | 1174 | input_hist = self.shell.input_hist |
|
1175 | 1175 | |
|
1176 | 1176 | if log_output: |
|
1177 | 1177 | log_write = logger.log_write |
|
1178 | 1178 | output_hist = self.shell.output_hist |
|
1179 | 1179 | for n in range(1,len(input_hist)-1): |
|
1180 | 1180 | log_write(input_hist[n].rstrip()) |
|
1181 | 1181 | if n in output_hist: |
|
1182 | 1182 | log_write(repr(output_hist[n]),'output') |
|
1183 | 1183 | else: |
|
1184 | 1184 | logger.log_write(input_hist[1:]) |
|
1185 | 1185 | if timestamp: |
|
1186 | 1186 | # re-enable timestamping |
|
1187 | 1187 | logger.timestamp = True |
|
1188 | 1188 | |
|
1189 | 1189 | print ('Activating auto-logging. ' |
|
1190 | 1190 | 'Current session state plus future input saved.') |
|
1191 | 1191 | logger.logstate() |
|
1192 | 1192 | |
|
1193 | 1193 | def magic_logstop(self,parameter_s=''): |
|
1194 | 1194 | """Fully stop logging and close log file. |
|
1195 | 1195 | |
|
1196 | 1196 | In order to start logging again, a new %logstart call needs to be made, |
|
1197 | 1197 | possibly (though not necessarily) with a new filename, mode and other |
|
1198 | 1198 | options.""" |
|
1199 | 1199 | self.logger.logstop() |
|
1200 | 1200 | |
|
1201 | 1201 | def magic_logoff(self,parameter_s=''): |
|
1202 | 1202 | """Temporarily stop logging. |
|
1203 | 1203 | |
|
1204 | 1204 | You must have previously started logging.""" |
|
1205 | 1205 | self.shell.logger.switch_log(0) |
|
1206 | 1206 | |
|
1207 | 1207 | def magic_logon(self,parameter_s=''): |
|
1208 | 1208 | """Restart logging. |
|
1209 | 1209 | |
|
1210 | 1210 | This function is for restarting logging which you've temporarily |
|
1211 | 1211 | stopped with %logoff. For starting logging for the first time, you |
|
1212 | 1212 | must use the %logstart function, which allows you to specify an |
|
1213 | 1213 | optional log filename.""" |
|
1214 | 1214 | |
|
1215 | 1215 | self.shell.logger.switch_log(1) |
|
1216 | 1216 | |
|
1217 | 1217 | def magic_logstate(self,parameter_s=''): |
|
1218 | 1218 | """Print the status of the logging system.""" |
|
1219 | 1219 | |
|
1220 | 1220 | self.shell.logger.logstate() |
|
1221 | 1221 | |
|
1222 | 1222 | def magic_pdb(self, parameter_s=''): |
|
1223 | 1223 | """Control the automatic calling of the pdb interactive debugger. |
|
1224 | 1224 | |
|
1225 | 1225 | Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without |
|
1226 | 1226 | argument it works as a toggle. |
|
1227 | 1227 | |
|
1228 | 1228 | When an exception is triggered, IPython can optionally call the |
|
1229 | 1229 | interactive pdb debugger after the traceback printout. %pdb toggles |
|
1230 | 1230 | this feature on and off. |
|
1231 | 1231 | |
|
1232 | 1232 | The initial state of this feature is set in your ipythonrc |
|
1233 | 1233 | configuration file (the variable is called 'pdb'). |
|
1234 | 1234 | |
|
1235 | 1235 | If you want to just activate the debugger AFTER an exception has fired, |
|
1236 | 1236 | without having to type '%pdb on' and rerunning your code, you can use |
|
1237 | 1237 | the %debug magic.""" |
|
1238 | 1238 | |
|
1239 | 1239 | par = parameter_s.strip().lower() |
|
1240 | 1240 | |
|
1241 | 1241 | if par: |
|
1242 | 1242 | try: |
|
1243 | 1243 | new_pdb = {'off':0,'0':0,'on':1,'1':1}[par] |
|
1244 | 1244 | except KeyError: |
|
1245 | 1245 | print ('Incorrect argument. Use on/1, off/0, ' |
|
1246 | 1246 | 'or nothing for a toggle.') |
|
1247 | 1247 | return |
|
1248 | 1248 | else: |
|
1249 | 1249 | # toggle |
|
1250 | 1250 | new_pdb = not self.shell.call_pdb |
|
1251 | 1251 | |
|
1252 | 1252 | # set on the shell |
|
1253 | 1253 | self.shell.call_pdb = new_pdb |
|
1254 | 1254 | print 'Automatic pdb calling has been turned',on_off(new_pdb) |
|
1255 | 1255 | |
|
1256 | 1256 | def magic_debug(self, parameter_s=''): |
|
1257 | 1257 | """Activate the interactive debugger in post-mortem mode. |
|
1258 | 1258 | |
|
1259 | 1259 | If an exception has just occurred, this lets you inspect its stack |
|
1260 | 1260 | frames interactively. Note that this will always work only on the last |
|
1261 | 1261 | traceback that occurred, so you must call this quickly after an |
|
1262 | 1262 | exception that you wish to inspect has fired, because if another one |
|
1263 | 1263 | occurs, it clobbers the previous one. |
|
1264 | 1264 | |
|
1265 | 1265 | If you want IPython to automatically do this on every exception, see |
|
1266 | 1266 | the %pdb magic for more details. |
|
1267 | 1267 | """ |
|
1268 | 1268 | |
|
1269 | 1269 | self.shell.debugger(force=True) |
|
1270 | 1270 | |
|
1271 | 1271 | @testdec.skip_doctest |
|
1272 | 1272 | def magic_prun(self, parameter_s ='',user_mode=1, |
|
1273 | 1273 | opts=None,arg_lst=None,prog_ns=None): |
|
1274 | 1274 | |
|
1275 | 1275 | """Run a statement through the python code profiler. |
|
1276 | 1276 | |
|
1277 | 1277 | Usage: |
|
1278 | 1278 | %prun [options] statement |
|
1279 | 1279 | |
|
1280 | 1280 | The given statement (which doesn't require quote marks) is run via the |
|
1281 | 1281 | python profiler in a manner similar to the profile.run() function. |
|
1282 | 1282 | Namespaces are internally managed to work correctly; profile.run |
|
1283 | 1283 | cannot be used in IPython because it makes certain assumptions about |
|
1284 | 1284 | namespaces which do not hold under IPython. |
|
1285 | 1285 | |
|
1286 | 1286 | Options: |
|
1287 | 1287 | |
|
1288 | 1288 | -l <limit>: you can place restrictions on what or how much of the |
|
1289 | 1289 | profile gets printed. The limit value can be: |
|
1290 | 1290 | |
|
1291 | 1291 | * A string: only information for function names containing this string |
|
1292 | 1292 | is printed. |
|
1293 | 1293 | |
|
1294 | 1294 | * An integer: only these many lines are printed. |
|
1295 | 1295 | |
|
1296 | 1296 | * A float (between 0 and 1): this fraction of the report is printed |
|
1297 | 1297 | (for example, use a limit of 0.4 to see the topmost 40% only). |
|
1298 | 1298 | |
|
1299 | 1299 | You can combine several limits with repeated use of the option. For |
|
1300 | 1300 | example, '-l __init__ -l 5' will print only the topmost 5 lines of |
|
1301 | 1301 | information about class constructors. |
|
1302 | 1302 | |
|
1303 | 1303 | -r: return the pstats.Stats object generated by the profiling. This |
|
1304 | 1304 | object has all the information about the profile in it, and you can |
|
1305 | 1305 | later use it for further analysis or in other functions. |
|
1306 | 1306 | |
|
1307 | 1307 | -s <key>: sort profile by given key. You can provide more than one key |
|
1308 | 1308 | by using the option several times: '-s key1 -s key2 -s key3...'. The |
|
1309 | 1309 | default sorting key is 'time'. |
|
1310 | 1310 | |
|
1311 | 1311 | The following is copied verbatim from the profile documentation |
|
1312 | 1312 | referenced below: |
|
1313 | 1313 | |
|
1314 | 1314 | When more than one key is provided, additional keys are used as |
|
1315 | 1315 | secondary criteria when the there is equality in all keys selected |
|
1316 | 1316 | before them. |
|
1317 | 1317 | |
|
1318 | 1318 | Abbreviations can be used for any key names, as long as the |
|
1319 | 1319 | abbreviation is unambiguous. The following are the keys currently |
|
1320 | 1320 | defined: |
|
1321 | 1321 | |
|
1322 | 1322 | Valid Arg Meaning |
|
1323 | 1323 | "calls" call count |
|
1324 | 1324 | "cumulative" cumulative time |
|
1325 | 1325 | "file" file name |
|
1326 | 1326 | "module" file name |
|
1327 | 1327 | "pcalls" primitive call count |
|
1328 | 1328 | "line" line number |
|
1329 | 1329 | "name" function name |
|
1330 | 1330 | "nfl" name/file/line |
|
1331 | 1331 | "stdname" standard name |
|
1332 | 1332 | "time" internal time |
|
1333 | 1333 | |
|
1334 | 1334 | Note that all sorts on statistics are in descending order (placing |
|
1335 | 1335 | most time consuming items first), where as name, file, and line number |
|
1336 | 1336 | searches are in ascending order (i.e., alphabetical). The subtle |
|
1337 | 1337 | distinction between "nfl" and "stdname" is that the standard name is a |
|
1338 | 1338 | sort of the name as printed, which means that the embedded line |
|
1339 | 1339 | numbers get compared in an odd way. For example, lines 3, 20, and 40 |
|
1340 | 1340 | would (if the file names were the same) appear in the string order |
|
1341 | 1341 | "20" "3" and "40". In contrast, "nfl" does a numeric compare of the |
|
1342 | 1342 | line numbers. In fact, sort_stats("nfl") is the same as |
|
1343 | 1343 | sort_stats("name", "file", "line"). |
|
1344 | 1344 | |
|
1345 | 1345 | -T <filename>: save profile results as shown on screen to a text |
|
1346 | 1346 | file. The profile is still shown on screen. |
|
1347 | 1347 | |
|
1348 | 1348 | -D <filename>: save (via dump_stats) profile statistics to given |
|
1349 | 1349 | filename. This data is in a format understod by the pstats module, and |
|
1350 | 1350 | is generated by a call to the dump_stats() method of profile |
|
1351 | 1351 | objects. The profile is still shown on screen. |
|
1352 | 1352 | |
|
1353 | 1353 | If you want to run complete programs under the profiler's control, use |
|
1354 | 1354 | '%run -p [prof_opts] filename.py [args to program]' where prof_opts |
|
1355 | 1355 | contains profiler specific options as described here. |
|
1356 | 1356 | |
|
1357 | 1357 | You can read the complete documentation for the profile module with:: |
|
1358 | 1358 | |
|
1359 | 1359 | In [1]: import profile; profile.help() |
|
1360 | 1360 | """ |
|
1361 | 1361 | |
|
1362 | 1362 | opts_def = Struct(D=[''],l=[],s=['time'],T=['']) |
|
1363 | 1363 | # protect user quote marks |
|
1364 | 1364 | parameter_s = parameter_s.replace('"',r'\"').replace("'",r"\'") |
|
1365 | 1365 | |
|
1366 | 1366 | if user_mode: # regular user call |
|
1367 | 1367 | opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:', |
|
1368 | 1368 | list_all=1) |
|
1369 | 1369 | namespace = self.shell.user_ns |
|
1370 | 1370 | else: # called to run a program by %run -p |
|
1371 | 1371 | try: |
|
1372 | 1372 | filename = get_py_filename(arg_lst[0]) |
|
1373 | 1373 | except IOError,msg: |
|
1374 | 1374 | error(msg) |
|
1375 | 1375 | return |
|
1376 | 1376 | |
|
1377 | 1377 | arg_str = 'execfile(filename,prog_ns)' |
|
1378 | 1378 | namespace = locals() |
|
1379 | 1379 | |
|
1380 | 1380 | opts.merge(opts_def) |
|
1381 | 1381 | |
|
1382 | 1382 | prof = profile.Profile() |
|
1383 | 1383 | try: |
|
1384 | 1384 | prof = prof.runctx(arg_str,namespace,namespace) |
|
1385 | 1385 | sys_exit = '' |
|
1386 | 1386 | except SystemExit: |
|
1387 | 1387 | sys_exit = """*** SystemExit exception caught in code being profiled.""" |
|
1388 | 1388 | |
|
1389 | 1389 | stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s) |
|
1390 | 1390 | |
|
1391 | 1391 | lims = opts.l |
|
1392 | 1392 | if lims: |
|
1393 | 1393 | lims = [] # rebuild lims with ints/floats/strings |
|
1394 | 1394 | for lim in opts.l: |
|
1395 | 1395 | try: |
|
1396 | 1396 | lims.append(int(lim)) |
|
1397 | 1397 | except ValueError: |
|
1398 | 1398 | try: |
|
1399 | 1399 | lims.append(float(lim)) |
|
1400 | 1400 | except ValueError: |
|
1401 | 1401 | lims.append(lim) |
|
1402 | 1402 | |
|
1403 | 1403 | # Trap output. |
|
1404 | 1404 | stdout_trap = StringIO() |
|
1405 | 1405 | |
|
1406 | 1406 | if hasattr(stats,'stream'): |
|
1407 | 1407 | # In newer versions of python, the stats object has a 'stream' |
|
1408 | 1408 | # attribute to write into. |
|
1409 | 1409 | stats.stream = stdout_trap |
|
1410 | 1410 | stats.print_stats(*lims) |
|
1411 | 1411 | else: |
|
1412 | 1412 | # For older versions, we manually redirect stdout during printing |
|
1413 | 1413 | sys_stdout = sys.stdout |
|
1414 | 1414 | try: |
|
1415 | 1415 | sys.stdout = stdout_trap |
|
1416 | 1416 | stats.print_stats(*lims) |
|
1417 | 1417 | finally: |
|
1418 | 1418 | sys.stdout = sys_stdout |
|
1419 | 1419 | |
|
1420 | 1420 | output = stdout_trap.getvalue() |
|
1421 | 1421 | output = output.rstrip() |
|
1422 | 1422 | |
|
1423 | 1423 | page(output,screen_lines=self.shell.rc.screen_length) |
|
1424 | 1424 | print sys_exit, |
|
1425 | 1425 | |
|
1426 | 1426 | dump_file = opts.D[0] |
|
1427 | 1427 | text_file = opts.T[0] |
|
1428 | 1428 | if dump_file: |
|
1429 | 1429 | prof.dump_stats(dump_file) |
|
1430 | 1430 | print '\n*** Profile stats marshalled to file',\ |
|
1431 | 1431 | `dump_file`+'.',sys_exit |
|
1432 | 1432 | if text_file: |
|
1433 | 1433 | pfile = file(text_file,'w') |
|
1434 | 1434 | pfile.write(output) |
|
1435 | 1435 | pfile.close() |
|
1436 | 1436 | print '\n*** Profile printout saved to text file',\ |
|
1437 | 1437 | `text_file`+'.',sys_exit |
|
1438 | 1438 | |
|
1439 | 1439 | if opts.has_key('r'): |
|
1440 | 1440 | return stats |
|
1441 | 1441 | else: |
|
1442 | 1442 | return None |
|
1443 | 1443 | |
|
1444 | 1444 | @testdec.skip_doctest |
|
1445 | 1445 | def magic_run(self, parameter_s ='',runner=None, |
|
1446 | 1446 | file_finder=get_py_filename): |
|
1447 | 1447 | """Run the named file inside IPython as a program. |
|
1448 | 1448 | |
|
1449 | 1449 | Usage:\\ |
|
1450 | 1450 | %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args] |
|
1451 | 1451 | |
|
1452 | 1452 | Parameters after the filename are passed as command-line arguments to |
|
1453 | 1453 | the program (put in sys.argv). Then, control returns to IPython's |
|
1454 | 1454 | prompt. |
|
1455 | 1455 | |
|
1456 | 1456 | This is similar to running at a system prompt:\\ |
|
1457 | 1457 | $ python file args\\ |
|
1458 | 1458 | but with the advantage of giving you IPython's tracebacks, and of |
|
1459 | 1459 | loading all variables into your interactive namespace for further use |
|
1460 | 1460 | (unless -p is used, see below). |
|
1461 | 1461 | |
|
1462 | 1462 | The file is executed in a namespace initially consisting only of |
|
1463 | 1463 | __name__=='__main__' and sys.argv constructed as indicated. It thus |
|
1464 | 1464 | sees its environment as if it were being run as a stand-alone program |
|
1465 | 1465 | (except for sharing global objects such as previously imported |
|
1466 | 1466 | modules). But after execution, the IPython interactive namespace gets |
|
1467 | 1467 | updated with all variables defined in the program (except for __name__ |
|
1468 | 1468 | and sys.argv). This allows for very convenient loading of code for |
|
1469 | 1469 | interactive work, while giving each program a 'clean sheet' to run in. |
|
1470 | 1470 | |
|
1471 | 1471 | Options: |
|
1472 | 1472 | |
|
1473 | 1473 | -n: __name__ is NOT set to '__main__', but to the running file's name |
|
1474 | 1474 | without extension (as python does under import). This allows running |
|
1475 | 1475 | scripts and reloading the definitions in them without calling code |
|
1476 | 1476 | protected by an ' if __name__ == "__main__" ' clause. |
|
1477 | 1477 | |
|
1478 | 1478 | -i: run the file in IPython's namespace instead of an empty one. This |
|
1479 | 1479 | is useful if you are experimenting with code written in a text editor |
|
1480 | 1480 | which depends on variables defined interactively. |
|
1481 | 1481 | |
|
1482 | 1482 | -e: ignore sys.exit() calls or SystemExit exceptions in the script |
|
1483 | 1483 | being run. This is particularly useful if IPython is being used to |
|
1484 | 1484 | run unittests, which always exit with a sys.exit() call. In such |
|
1485 | 1485 | cases you are interested in the output of the test results, not in |
|
1486 | 1486 | seeing a traceback of the unittest module. |
|
1487 | 1487 | |
|
1488 | 1488 | -t: print timing information at the end of the run. IPython will give |
|
1489 | 1489 | you an estimated CPU time consumption for your script, which under |
|
1490 | 1490 | Unix uses the resource module to avoid the wraparound problems of |
|
1491 | 1491 | time.clock(). Under Unix, an estimate of time spent on system tasks |
|
1492 | 1492 | is also given (for Windows platforms this is reported as 0.0). |
|
1493 | 1493 | |
|
1494 | 1494 | If -t is given, an additional -N<N> option can be given, where <N> |
|
1495 | 1495 | must be an integer indicating how many times you want the script to |
|
1496 | 1496 | run. The final timing report will include total and per run results. |
|
1497 | 1497 | |
|
1498 | 1498 | For example (testing the script uniq_stable.py): |
|
1499 | 1499 | |
|
1500 | 1500 | In [1]: run -t uniq_stable |
|
1501 | 1501 | |
|
1502 | 1502 | IPython CPU timings (estimated):\\ |
|
1503 | 1503 | User : 0.19597 s.\\ |
|
1504 | 1504 | System: 0.0 s.\\ |
|
1505 | 1505 | |
|
1506 | 1506 | In [2]: run -t -N5 uniq_stable |
|
1507 | 1507 | |
|
1508 | 1508 | IPython CPU timings (estimated):\\ |
|
1509 | 1509 | Total runs performed: 5\\ |
|
1510 | 1510 | Times : Total Per run\\ |
|
1511 | 1511 | User : 0.910862 s, 0.1821724 s.\\ |
|
1512 | 1512 | System: 0.0 s, 0.0 s. |
|
1513 | 1513 | |
|
1514 | 1514 | -d: run your program under the control of pdb, the Python debugger. |
|
1515 | 1515 | This allows you to execute your program step by step, watch variables, |
|
1516 | 1516 | etc. Internally, what IPython does is similar to calling: |
|
1517 | 1517 | |
|
1518 | 1518 | pdb.run('execfile("YOURFILENAME")') |
|
1519 | 1519 | |
|
1520 | 1520 | with a breakpoint set on line 1 of your file. You can change the line |
|
1521 | 1521 | number for this automatic breakpoint to be <N> by using the -bN option |
|
1522 | 1522 | (where N must be an integer). For example: |
|
1523 | 1523 | |
|
1524 | 1524 | %run -d -b40 myscript |
|
1525 | 1525 | |
|
1526 | 1526 | will set the first breakpoint at line 40 in myscript.py. Note that |
|
1527 | 1527 | the first breakpoint must be set on a line which actually does |
|
1528 | 1528 | something (not a comment or docstring) for it to stop execution. |
|
1529 | 1529 | |
|
1530 | 1530 | When the pdb debugger starts, you will see a (Pdb) prompt. You must |
|
1531 | 1531 | first enter 'c' (without qoutes) to start execution up to the first |
|
1532 | 1532 | breakpoint. |
|
1533 | 1533 | |
|
1534 | 1534 | Entering 'help' gives information about the use of the debugger. You |
|
1535 | 1535 | can easily see pdb's full documentation with "import pdb;pdb.help()" |
|
1536 | 1536 | at a prompt. |
|
1537 | 1537 | |
|
1538 | 1538 | -p: run program under the control of the Python profiler module (which |
|
1539 | 1539 | prints a detailed report of execution times, function calls, etc). |
|
1540 | 1540 | |
|
1541 | 1541 | You can pass other options after -p which affect the behavior of the |
|
1542 | 1542 | profiler itself. See the docs for %prun for details. |
|
1543 | 1543 | |
|
1544 | 1544 | In this mode, the program's variables do NOT propagate back to the |
|
1545 | 1545 | IPython interactive namespace (because they remain in the namespace |
|
1546 | 1546 | where the profiler executes them). |
|
1547 | 1547 | |
|
1548 | 1548 | Internally this triggers a call to %prun, see its documentation for |
|
1549 | 1549 | details on the options available specifically for profiling. |
|
1550 | 1550 | |
|
1551 | 1551 | There is one special usage for which the text above doesn't apply: |
|
1552 | 1552 | if the filename ends with .ipy, the file is run as ipython script, |
|
1553 | 1553 | just as if the commands were written on IPython prompt. |
|
1554 | 1554 | """ |
|
1555 | 1555 | |
|
1556 | 1556 | # get arguments and set sys.argv for program to be run. |
|
1557 | 1557 | opts,arg_lst = self.parse_options(parameter_s,'nidtN:b:pD:l:rs:T:e', |
|
1558 | 1558 | mode='list',list_all=1) |
|
1559 | 1559 | |
|
1560 | 1560 | try: |
|
1561 | 1561 | filename = file_finder(arg_lst[0]) |
|
1562 | 1562 | except IndexError: |
|
1563 | 1563 | warn('you must provide at least a filename.') |
|
1564 | 1564 | print '\n%run:\n',OInspect.getdoc(self.magic_run) |
|
1565 | 1565 | return |
|
1566 | 1566 | except IOError,msg: |
|
1567 | 1567 | error(msg) |
|
1568 | 1568 | return |
|
1569 | 1569 | |
|
1570 | 1570 | if filename.lower().endswith('.ipy'): |
|
1571 | 1571 | self.api.runlines(open(filename).read()) |
|
1572 | 1572 | return |
|
1573 | 1573 | |
|
1574 | 1574 | # Control the response to exit() calls made by the script being run |
|
1575 | 1575 | exit_ignore = opts.has_key('e') |
|
1576 | 1576 | |
|
1577 | 1577 | # Make sure that the running script gets a proper sys.argv as if it |
|
1578 | 1578 | # were run from a system shell. |
|
1579 | 1579 | save_argv = sys.argv # save it for later restoring |
|
1580 | 1580 | sys.argv = [filename]+ arg_lst[1:] # put in the proper filename |
|
1581 | 1581 | |
|
1582 | 1582 | if opts.has_key('i'): |
|
1583 | 1583 | # Run in user's interactive namespace |
|
1584 | 1584 | prog_ns = self.shell.user_ns |
|
1585 | 1585 | __name__save = self.shell.user_ns['__name__'] |
|
1586 | 1586 | prog_ns['__name__'] = '__main__' |
|
1587 |
main_mod = |
|
|
1587 | main_mod = self.shell.new_main_mod(prog_ns) | |
|
1588 | 1588 | else: |
|
1589 | 1589 | # Run in a fresh, empty namespace |
|
1590 | 1590 | if opts.has_key('n'): |
|
1591 | 1591 | name = os.path.splitext(os.path.basename(filename))[0] |
|
1592 | 1592 | else: |
|
1593 | 1593 | name = '__main__' |
|
1594 | main_mod = FakeModule() | |
|
1594 | ||
|
1595 | main_mod = self.shell.new_main_mod() | |
|
1595 | 1596 | prog_ns = main_mod.__dict__ |
|
1596 | 1597 | prog_ns['__name__'] = name |
|
1597 | ||
|
1598 | # The shell MUST hold a reference to main_mod so after %run exits, | |
|
1599 | # the python deletion mechanism doesn't zero it out (leaving | |
|
1600 | # dangling references). However, we should drop old versions of | |
|
1601 | # main_mod. There is now a proper API to manage this caching in | |
|
1602 | # the main shell object, we use that. | |
|
1603 | self.shell.cache_main_mod(main_mod) | |
|
1604 | 1598 | |
|
1605 | 1599 | # Since '%run foo' emulates 'python foo.py' at the cmd line, we must |
|
1606 | 1600 | # set the __file__ global in the script's namespace |
|
1607 | 1601 | prog_ns['__file__'] = filename |
|
1608 | 1602 | |
|
1609 | 1603 | # pickle fix. See iplib for an explanation. But we need to make sure |
|
1610 | 1604 | # that, if we overwrite __main__, we replace it at the end |
|
1611 | 1605 | main_mod_name = prog_ns['__name__'] |
|
1612 | 1606 | |
|
1613 | 1607 | if main_mod_name == '__main__': |
|
1614 | 1608 | restore_main = sys.modules['__main__'] |
|
1615 | 1609 | else: |
|
1616 | 1610 | restore_main = False |
|
1617 | 1611 | |
|
1618 | 1612 | # This needs to be undone at the end to prevent holding references to |
|
1619 | 1613 | # every single object ever created. |
|
1620 | 1614 | sys.modules[main_mod_name] = main_mod |
|
1621 | 1615 | |
|
1622 | 1616 | stats = None |
|
1623 | 1617 | try: |
|
1624 | 1618 | self.shell.savehist() |
|
1625 | 1619 | |
|
1626 | 1620 | if opts.has_key('p'): |
|
1627 | 1621 | stats = self.magic_prun('',0,opts,arg_lst,prog_ns) |
|
1628 | 1622 | else: |
|
1629 | 1623 | if opts.has_key('d'): |
|
1630 | 1624 | deb = Debugger.Pdb(self.shell.rc.colors) |
|
1631 | 1625 | # reset Breakpoint state, which is moronically kept |
|
1632 | 1626 | # in a class |
|
1633 | 1627 | bdb.Breakpoint.next = 1 |
|
1634 | 1628 | bdb.Breakpoint.bplist = {} |
|
1635 | 1629 | bdb.Breakpoint.bpbynumber = [None] |
|
1636 | 1630 | # Set an initial breakpoint to stop execution |
|
1637 | 1631 | maxtries = 10 |
|
1638 | 1632 | bp = int(opts.get('b',[1])[0]) |
|
1639 | 1633 | checkline = deb.checkline(filename,bp) |
|
1640 | 1634 | if not checkline: |
|
1641 | 1635 | for bp in range(bp+1,bp+maxtries+1): |
|
1642 | 1636 | if deb.checkline(filename,bp): |
|
1643 | 1637 | break |
|
1644 | 1638 | else: |
|
1645 | 1639 | msg = ("\nI failed to find a valid line to set " |
|
1646 | 1640 | "a breakpoint\n" |
|
1647 | 1641 | "after trying up to line: %s.\n" |
|
1648 | 1642 | "Please set a valid breakpoint manually " |
|
1649 | 1643 | "with the -b option." % bp) |
|
1650 | 1644 | error(msg) |
|
1651 | 1645 | return |
|
1652 | 1646 | # if we find a good linenumber, set the breakpoint |
|
1653 | 1647 | deb.do_break('%s:%s' % (filename,bp)) |
|
1654 | 1648 | # Start file run |
|
1655 | 1649 | print "NOTE: Enter 'c' at the", |
|
1656 | 1650 | print "%s prompt to start your script." % deb.prompt |
|
1657 | 1651 | try: |
|
1658 | 1652 | deb.run('execfile("%s")' % filename,prog_ns) |
|
1659 | 1653 | |
|
1660 | 1654 | except: |
|
1661 | 1655 | etype, value, tb = sys.exc_info() |
|
1662 | 1656 | # Skip three frames in the traceback: the %run one, |
|
1663 | 1657 | # one inside bdb.py, and the command-line typed by the |
|
1664 | 1658 | # user (run by exec in pdb itself). |
|
1665 | 1659 | self.shell.InteractiveTB(etype,value,tb,tb_offset=3) |
|
1666 | 1660 | else: |
|
1667 | 1661 | if runner is None: |
|
1668 | 1662 | runner = self.shell.safe_execfile |
|
1669 | 1663 | if opts.has_key('t'): |
|
1670 | 1664 | # timed execution |
|
1671 | 1665 | try: |
|
1672 | 1666 | nruns = int(opts['N'][0]) |
|
1673 | 1667 | if nruns < 1: |
|
1674 | 1668 | error('Number of runs must be >=1') |
|
1675 | 1669 | return |
|
1676 | 1670 | except (KeyError): |
|
1677 | 1671 | nruns = 1 |
|
1678 | 1672 | if nruns == 1: |
|
1679 | 1673 | t0 = clock2() |
|
1680 | 1674 | runner(filename,prog_ns,prog_ns, |
|
1681 | 1675 | exit_ignore=exit_ignore) |
|
1682 | 1676 | t1 = clock2() |
|
1683 | 1677 | t_usr = t1[0]-t0[0] |
|
1684 | 1678 | t_sys = t1[1]-t1[1] |
|
1685 | 1679 | print "\nIPython CPU timings (estimated):" |
|
1686 | 1680 | print " User : %10s s." % t_usr |
|
1687 | 1681 | print " System: %10s s." % t_sys |
|
1688 | 1682 | else: |
|
1689 | 1683 | runs = range(nruns) |
|
1690 | 1684 | t0 = clock2() |
|
1691 | 1685 | for nr in runs: |
|
1692 | 1686 | runner(filename,prog_ns,prog_ns, |
|
1693 | 1687 | exit_ignore=exit_ignore) |
|
1694 | 1688 | t1 = clock2() |
|
1695 | 1689 | t_usr = t1[0]-t0[0] |
|
1696 | 1690 | t_sys = t1[1]-t1[1] |
|
1697 | 1691 | print "\nIPython CPU timings (estimated):" |
|
1698 | 1692 | print "Total runs performed:",nruns |
|
1699 | 1693 | print " Times : %10s %10s" % ('Total','Per run') |
|
1700 | 1694 | print " User : %10s s, %10s s." % (t_usr,t_usr/nruns) |
|
1701 | 1695 | print " System: %10s s, %10s s." % (t_sys,t_sys/nruns) |
|
1702 | 1696 | |
|
1703 | 1697 | else: |
|
1704 | 1698 | # regular execution |
|
1705 | 1699 | runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore) |
|
1700 | ||
|
1706 | 1701 | if opts.has_key('i'): |
|
1707 | 1702 | self.shell.user_ns['__name__'] = __name__save |
|
1708 | 1703 | else: |
|
1704 | # The shell MUST hold a reference to prog_ns so after %run | |
|
1705 | # exits, the python deletion mechanism doesn't zero it out | |
|
1706 | # (leaving dangling references). | |
|
1707 | self.shell.cache_main_mod(prog_ns,filename) | |
|
1709 | 1708 | # update IPython interactive namespace |
|
1710 | 1709 | del prog_ns['__name__'] |
|
1711 | 1710 | self.shell.user_ns.update(prog_ns) |
|
1712 | 1711 | finally: |
|
1713 | 1712 | # Ensure key global structures are restored |
|
1714 | 1713 | sys.argv = save_argv |
|
1715 | 1714 | if restore_main: |
|
1716 | 1715 | sys.modules['__main__'] = restore_main |
|
1717 | 1716 | else: |
|
1718 | 1717 | # Remove from sys.modules the reference to main_mod we'd |
|
1719 | 1718 | # added. Otherwise it will trap references to objects |
|
1720 | 1719 | # contained therein. |
|
1721 | 1720 | del sys.modules[main_mod_name] |
|
1721 | ||
|
1722 | 1722 | self.shell.reloadhist() |
|
1723 | 1723 | |
|
1724 | 1724 | return stats |
|
1725 | 1725 | |
|
1726 | 1726 | def magic_runlog(self, parameter_s =''): |
|
1727 | 1727 | """Run files as logs. |
|
1728 | 1728 | |
|
1729 | 1729 | Usage:\\ |
|
1730 | 1730 | %runlog file1 file2 ... |
|
1731 | 1731 | |
|
1732 | 1732 | Run the named files (treating them as log files) in sequence inside |
|
1733 | 1733 | the interpreter, and return to the prompt. This is much slower than |
|
1734 | 1734 | %run because each line is executed in a try/except block, but it |
|
1735 | 1735 | allows running files with syntax errors in them. |
|
1736 | 1736 | |
|
1737 | 1737 | Normally IPython will guess when a file is one of its own logfiles, so |
|
1738 | 1738 | you can typically use %run even for logs. This shorthand allows you to |
|
1739 | 1739 | force any file to be treated as a log file.""" |
|
1740 | 1740 | |
|
1741 | 1741 | for f in parameter_s.split(): |
|
1742 | 1742 | self.shell.safe_execfile(f,self.shell.user_ns, |
|
1743 | 1743 | self.shell.user_ns,islog=1) |
|
1744 | 1744 | |
|
1745 | 1745 | @testdec.skip_doctest |
|
1746 | 1746 | def magic_timeit(self, parameter_s =''): |
|
1747 | 1747 | """Time execution of a Python statement or expression |
|
1748 | 1748 | |
|
1749 | 1749 | Usage:\\ |
|
1750 | 1750 | %timeit [-n<N> -r<R> [-t|-c]] statement |
|
1751 | 1751 | |
|
1752 | 1752 | Time execution of a Python statement or expression using the timeit |
|
1753 | 1753 | module. |
|
1754 | 1754 | |
|
1755 | 1755 | Options: |
|
1756 | 1756 | -n<N>: execute the given statement <N> times in a loop. If this value |
|
1757 | 1757 | is not given, a fitting value is chosen. |
|
1758 | 1758 | |
|
1759 | 1759 | -r<R>: repeat the loop iteration <R> times and take the best result. |
|
1760 | 1760 | Default: 3 |
|
1761 | 1761 | |
|
1762 | 1762 | -t: use time.time to measure the time, which is the default on Unix. |
|
1763 | 1763 | This function measures wall time. |
|
1764 | 1764 | |
|
1765 | 1765 | -c: use time.clock to measure the time, which is the default on |
|
1766 | 1766 | Windows and measures wall time. On Unix, resource.getrusage is used |
|
1767 | 1767 | instead and returns the CPU user time. |
|
1768 | 1768 | |
|
1769 | 1769 | -p<P>: use a precision of <P> digits to display the timing result. |
|
1770 | 1770 | Default: 3 |
|
1771 | 1771 | |
|
1772 | 1772 | |
|
1773 | 1773 | Examples: |
|
1774 | 1774 | |
|
1775 | 1775 | In [1]: %timeit pass |
|
1776 | 1776 | 10000000 loops, best of 3: 53.3 ns per loop |
|
1777 | 1777 | |
|
1778 | 1778 | In [2]: u = None |
|
1779 | 1779 | |
|
1780 | 1780 | In [3]: %timeit u is None |
|
1781 | 1781 | 10000000 loops, best of 3: 184 ns per loop |
|
1782 | 1782 | |
|
1783 | 1783 | In [4]: %timeit -r 4 u == None |
|
1784 | 1784 | 1000000 loops, best of 4: 242 ns per loop |
|
1785 | 1785 | |
|
1786 | 1786 | In [5]: import time |
|
1787 | 1787 | |
|
1788 | 1788 | In [6]: %timeit -n1 time.sleep(2) |
|
1789 | 1789 | 1 loops, best of 3: 2 s per loop |
|
1790 | 1790 | |
|
1791 | 1791 | |
|
1792 | 1792 | The times reported by %timeit will be slightly higher than those |
|
1793 | 1793 | reported by the timeit.py script when variables are accessed. This is |
|
1794 | 1794 | due to the fact that %timeit executes the statement in the namespace |
|
1795 | 1795 | of the shell, compared with timeit.py, which uses a single setup |
|
1796 | 1796 | statement to import function or create variables. Generally, the bias |
|
1797 | 1797 | does not matter as long as results from timeit.py are not mixed with |
|
1798 | 1798 | those from %timeit.""" |
|
1799 | 1799 | |
|
1800 | 1800 | import timeit |
|
1801 | 1801 | import math |
|
1802 | 1802 | |
|
1803 | units = [u"s", u"ms", u"\xb5s", u"ns"] | |
|
1803 | # XXX: Unfortunately the unicode 'micro' symbol can cause problems in | |
|
1804 | # certain terminals. Until we figure out a robust way of | |
|
1805 | # auto-detecting if the terminal can deal with it, use plain 'us' for | |
|
1806 | # microseconds. I am really NOT happy about disabling the proper | |
|
1807 | # 'micro' prefix, but crashing is worse... If anyone knows what the | |
|
1808 | # right solution for this is, I'm all ears... | |
|
1809 | # | |
|
1810 | # Note: using | |
|
1811 | # | |
|
1812 | # s = u'\xb5' | |
|
1813 | # s.encode(sys.getdefaultencoding()) | |
|
1814 | # | |
|
1815 | # is not sufficient, as I've seen terminals where that fails but | |
|
1816 | # print s | |
|
1817 | # | |
|
1818 | # succeeds | |
|
1819 | # | |
|
1820 | # See bug: https://bugs.launchpad.net/ipython/+bug/348466 | |
|
1821 | ||
|
1822 | #units = [u"s", u"ms",u'\xb5',"ns"] | |
|
1823 | units = [u"s", u"ms",u'us',"ns"] | |
|
1824 | ||
|
1804 | 1825 | scaling = [1, 1e3, 1e6, 1e9] |
|
1805 | 1826 | |
|
1806 | 1827 | opts, stmt = self.parse_options(parameter_s,'n:r:tcp:', |
|
1807 | 1828 | posix=False) |
|
1808 | 1829 | if stmt == "": |
|
1809 | 1830 | return |
|
1810 | 1831 | timefunc = timeit.default_timer |
|
1811 | 1832 | number = int(getattr(opts, "n", 0)) |
|
1812 | 1833 | repeat = int(getattr(opts, "r", timeit.default_repeat)) |
|
1813 | 1834 | precision = int(getattr(opts, "p", 3)) |
|
1814 | 1835 | if hasattr(opts, "t"): |
|
1815 | 1836 | timefunc = time.time |
|
1816 | 1837 | if hasattr(opts, "c"): |
|
1817 | 1838 | timefunc = clock |
|
1818 | 1839 | |
|
1819 | 1840 | timer = timeit.Timer(timer=timefunc) |
|
1820 | 1841 | # this code has tight coupling to the inner workings of timeit.Timer, |
|
1821 | 1842 | # but is there a better way to achieve that the code stmt has access |
|
1822 | 1843 | # to the shell namespace? |
|
1823 | 1844 | |
|
1824 | 1845 | src = timeit.template % {'stmt': timeit.reindent(stmt, 8), |
|
1825 | 1846 | 'setup': "pass"} |
|
1826 | 1847 | # Track compilation time so it can be reported if too long |
|
1827 | 1848 | # Minimum time above which compilation time will be reported |
|
1828 | 1849 | tc_min = 0.1 |
|
1829 | 1850 | |
|
1830 | 1851 | t0 = clock() |
|
1831 | 1852 | code = compile(src, "<magic-timeit>", "exec") |
|
1832 | 1853 | tc = clock()-t0 |
|
1833 | 1854 | |
|
1834 | 1855 | ns = {} |
|
1835 | 1856 | exec code in self.shell.user_ns, ns |
|
1836 | 1857 | timer.inner = ns["inner"] |
|
1837 | 1858 | |
|
1838 | 1859 | if number == 0: |
|
1839 | 1860 | # determine number so that 0.2 <= total time < 2.0 |
|
1840 | 1861 | number = 1 |
|
1841 | 1862 | for i in range(1, 10): |
|
1842 | number *= 10 | |
|
1843 | 1863 | if timer.timeit(number) >= 0.2: |
|
1844 | 1864 | break |
|
1865 | number *= 10 | |
|
1845 | 1866 | |
|
1846 | 1867 | best = min(timer.repeat(repeat, number)) / number |
|
1847 | 1868 | |
|
1848 | 1869 | if best > 0.0: |
|
1849 | 1870 | order = min(-int(math.floor(math.log10(best)) // 3), 3) |
|
1850 | 1871 | else: |
|
1851 | 1872 | order = 3 |
|
1852 | 1873 | print u"%d loops, best of %d: %.*g %s per loop" % (number, repeat, |
|
1853 | 1874 | precision, |
|
1854 | 1875 | best * scaling[order], |
|
1855 | 1876 | units[order]) |
|
1856 | 1877 | if tc > tc_min: |
|
1857 | 1878 | print "Compiler time: %.2f s" % tc |
|
1858 | 1879 | |
|
1859 | 1880 | @testdec.skip_doctest |
|
1860 | 1881 | def magic_time(self,parameter_s = ''): |
|
1861 | 1882 | """Time execution of a Python statement or expression. |
|
1862 | 1883 | |
|
1863 | 1884 | The CPU and wall clock times are printed, and the value of the |
|
1864 | 1885 | expression (if any) is returned. Note that under Win32, system time |
|
1865 | 1886 | is always reported as 0, since it can not be measured. |
|
1866 | 1887 | |
|
1867 | 1888 | This function provides very basic timing functionality. In Python |
|
1868 | 1889 | 2.3, the timeit module offers more control and sophistication, so this |
|
1869 | 1890 | could be rewritten to use it (patches welcome). |
|
1870 | 1891 | |
|
1871 | 1892 | Some examples: |
|
1872 | 1893 | |
|
1873 | 1894 | In [1]: time 2**128 |
|
1874 | 1895 | CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s |
|
1875 | 1896 | Wall time: 0.00 |
|
1876 | 1897 | Out[1]: 340282366920938463463374607431768211456L |
|
1877 | 1898 | |
|
1878 | 1899 | In [2]: n = 1000000 |
|
1879 | 1900 | |
|
1880 | 1901 | In [3]: time sum(range(n)) |
|
1881 | 1902 | CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s |
|
1882 | 1903 | Wall time: 1.37 |
|
1883 | 1904 | Out[3]: 499999500000L |
|
1884 | 1905 | |
|
1885 | 1906 | In [4]: time print 'hello world' |
|
1886 | 1907 | hello world |
|
1887 | 1908 | CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s |
|
1888 | 1909 | Wall time: 0.00 |
|
1889 | 1910 | |
|
1890 | 1911 | Note that the time needed by Python to compile the given expression |
|
1891 | 1912 | will be reported if it is more than 0.1s. In this example, the |
|
1892 | 1913 | actual exponentiation is done by Python at compilation time, so while |
|
1893 | 1914 | the expression can take a noticeable amount of time to compute, that |
|
1894 | 1915 | time is purely due to the compilation: |
|
1895 | 1916 | |
|
1896 | 1917 | In [5]: time 3**9999; |
|
1897 | 1918 | CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s |
|
1898 | 1919 | Wall time: 0.00 s |
|
1899 | 1920 | |
|
1900 | 1921 | In [6]: time 3**999999; |
|
1901 | 1922 | CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s |
|
1902 | 1923 | Wall time: 0.00 s |
|
1903 | 1924 | Compiler : 0.78 s |
|
1904 | 1925 | """ |
|
1905 | 1926 | |
|
1906 | 1927 | # fail immediately if the given expression can't be compiled |
|
1907 | 1928 | |
|
1908 | 1929 | expr = self.shell.prefilter(parameter_s,False) |
|
1909 | 1930 | |
|
1910 | 1931 | # Minimum time above which compilation time will be reported |
|
1911 | 1932 | tc_min = 0.1 |
|
1912 | 1933 | |
|
1913 | 1934 | try: |
|
1914 | 1935 | mode = 'eval' |
|
1915 | 1936 | t0 = clock() |
|
1916 | 1937 | code = compile(expr,'<timed eval>',mode) |
|
1917 | 1938 | tc = clock()-t0 |
|
1918 | 1939 | except SyntaxError: |
|
1919 | 1940 | mode = 'exec' |
|
1920 | 1941 | t0 = clock() |
|
1921 | 1942 | code = compile(expr,'<timed exec>',mode) |
|
1922 | 1943 | tc = clock()-t0 |
|
1923 | 1944 | # skew measurement as little as possible |
|
1924 | 1945 | glob = self.shell.user_ns |
|
1925 | 1946 | clk = clock2 |
|
1926 | 1947 | wtime = time.time |
|
1927 | 1948 | # time execution |
|
1928 | 1949 | wall_st = wtime() |
|
1929 | 1950 | if mode=='eval': |
|
1930 | 1951 | st = clk() |
|
1931 | 1952 | out = eval(code,glob) |
|
1932 | 1953 | end = clk() |
|
1933 | 1954 | else: |
|
1934 | 1955 | st = clk() |
|
1935 | 1956 | exec code in glob |
|
1936 | 1957 | end = clk() |
|
1937 | 1958 | out = None |
|
1938 | 1959 | wall_end = wtime() |
|
1939 | 1960 | # Compute actual times and report |
|
1940 | 1961 | wall_time = wall_end-wall_st |
|
1941 | 1962 | cpu_user = end[0]-st[0] |
|
1942 | 1963 | cpu_sys = end[1]-st[1] |
|
1943 | 1964 | cpu_tot = cpu_user+cpu_sys |
|
1944 | 1965 | print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \ |
|
1945 | 1966 | (cpu_user,cpu_sys,cpu_tot) |
|
1946 | 1967 | print "Wall time: %.2f s" % wall_time |
|
1947 | 1968 | if tc > tc_min: |
|
1948 | 1969 | print "Compiler : %.2f s" % tc |
|
1949 | 1970 | return out |
|
1950 | 1971 | |
|
1951 | 1972 | @testdec.skip_doctest |
|
1952 | 1973 | def magic_macro(self,parameter_s = ''): |
|
1953 | 1974 | """Define a set of input lines as a macro for future re-execution. |
|
1954 | 1975 | |
|
1955 | 1976 | Usage:\\ |
|
1956 | 1977 | %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ... |
|
1957 | 1978 | |
|
1958 | 1979 | Options: |
|
1959 | 1980 | |
|
1960 | 1981 | -r: use 'raw' input. By default, the 'processed' history is used, |
|
1961 | 1982 | so that magics are loaded in their transformed version to valid |
|
1962 | 1983 | Python. If this option is given, the raw input as typed as the |
|
1963 | 1984 | command line is used instead. |
|
1964 | 1985 | |
|
1965 | 1986 | This will define a global variable called `name` which is a string |
|
1966 | 1987 | made of joining the slices and lines you specify (n1,n2,... numbers |
|
1967 | 1988 | above) from your input history into a single string. This variable |
|
1968 | 1989 | acts like an automatic function which re-executes those lines as if |
|
1969 | 1990 | you had typed them. You just type 'name' at the prompt and the code |
|
1970 | 1991 | executes. |
|
1971 | 1992 | |
|
1972 | 1993 | The notation for indicating number ranges is: n1-n2 means 'use line |
|
1973 | 1994 | numbers n1,...n2' (the endpoint is included). That is, '5-7' means |
|
1974 | 1995 | using the lines numbered 5,6 and 7. |
|
1975 | 1996 | |
|
1976 | 1997 | Note: as a 'hidden' feature, you can also use traditional python slice |
|
1977 | 1998 | notation, where N:M means numbers N through M-1. |
|
1978 | 1999 | |
|
1979 | 2000 | For example, if your history contains (%hist prints it): |
|
1980 | 2001 | |
|
1981 | 2002 | 44: x=1 |
|
1982 | 2003 | 45: y=3 |
|
1983 | 2004 | 46: z=x+y |
|
1984 | 2005 | 47: print x |
|
1985 | 2006 | 48: a=5 |
|
1986 | 2007 | 49: print 'x',x,'y',y |
|
1987 | 2008 | |
|
1988 | 2009 | you can create a macro with lines 44 through 47 (included) and line 49 |
|
1989 | 2010 | called my_macro with: |
|
1990 | 2011 | |
|
1991 | 2012 | In [55]: %macro my_macro 44-47 49 |
|
1992 | 2013 | |
|
1993 | 2014 | Now, typing `my_macro` (without quotes) will re-execute all this code |
|
1994 | 2015 | in one pass. |
|
1995 | 2016 | |
|
1996 | 2017 | You don't need to give the line-numbers in order, and any given line |
|
1997 | 2018 | number can appear multiple times. You can assemble macros with any |
|
1998 | 2019 | lines from your input history in any order. |
|
1999 | 2020 | |
|
2000 | 2021 | The macro is a simple object which holds its value in an attribute, |
|
2001 | 2022 | but IPython's display system checks for macros and executes them as |
|
2002 | 2023 | code instead of printing them when you type their name. |
|
2003 | 2024 | |
|
2004 | 2025 | You can view a macro's contents by explicitly printing it with: |
|
2005 | 2026 | |
|
2006 | 2027 | 'print macro_name'. |
|
2007 | 2028 | |
|
2008 | 2029 | For one-off cases which DON'T contain magic function calls in them you |
|
2009 | 2030 | can obtain similar results by explicitly executing slices from your |
|
2010 | 2031 | input history with: |
|
2011 | 2032 | |
|
2012 | 2033 | In [60]: exec In[44:48]+In[49]""" |
|
2013 | 2034 | |
|
2014 | 2035 | opts,args = self.parse_options(parameter_s,'r',mode='list') |
|
2015 | 2036 | if not args: |
|
2016 | 2037 | macs = [k for k,v in self.shell.user_ns.items() if isinstance(v, Macro)] |
|
2017 | 2038 | macs.sort() |
|
2018 | 2039 | return macs |
|
2019 | 2040 | if len(args) == 1: |
|
2020 | 2041 | raise UsageError( |
|
2021 | 2042 | "%macro insufficient args; usage '%macro name n1-n2 n3-4...") |
|
2022 | 2043 | name,ranges = args[0], args[1:] |
|
2023 | 2044 | |
|
2024 | 2045 | #print 'rng',ranges # dbg |
|
2025 | 2046 | lines = self.extract_input_slices(ranges,opts.has_key('r')) |
|
2026 | 2047 | macro = Macro(lines) |
|
2027 | 2048 | self.shell.user_ns.update({name:macro}) |
|
2028 | 2049 | print 'Macro `%s` created. To execute, type its name (without quotes).' % name |
|
2029 | 2050 | print 'Macro contents:' |
|
2030 | 2051 | print macro, |
|
2031 | 2052 | |
|
2032 | 2053 | def magic_save(self,parameter_s = ''): |
|
2033 | 2054 | """Save a set of lines to a given filename. |
|
2034 | 2055 | |
|
2035 | 2056 | Usage:\\ |
|
2036 | 2057 | %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ... |
|
2037 | 2058 | |
|
2038 | 2059 | Options: |
|
2039 | 2060 | |
|
2040 | 2061 | -r: use 'raw' input. By default, the 'processed' history is used, |
|
2041 | 2062 | so that magics are loaded in their transformed version to valid |
|
2042 | 2063 | Python. If this option is given, the raw input as typed as the |
|
2043 | 2064 | command line is used instead. |
|
2044 | 2065 | |
|
2045 | 2066 | This function uses the same syntax as %macro for line extraction, but |
|
2046 | 2067 | instead of creating a macro it saves the resulting string to the |
|
2047 | 2068 | filename you specify. |
|
2048 | 2069 | |
|
2049 | 2070 | It adds a '.py' extension to the file if you don't do so yourself, and |
|
2050 | 2071 | it asks for confirmation before overwriting existing files.""" |
|
2051 | 2072 | |
|
2052 | 2073 | opts,args = self.parse_options(parameter_s,'r',mode='list') |
|
2053 | 2074 | fname,ranges = args[0], args[1:] |
|
2054 | 2075 | if not fname.endswith('.py'): |
|
2055 | 2076 | fname += '.py' |
|
2056 | 2077 | if os.path.isfile(fname): |
|
2057 | 2078 | ans = raw_input('File `%s` exists. Overwrite (y/[N])? ' % fname) |
|
2058 | 2079 | if ans.lower() not in ['y','yes']: |
|
2059 | 2080 | print 'Operation cancelled.' |
|
2060 | 2081 | return |
|
2061 | 2082 | cmds = ''.join(self.extract_input_slices(ranges,opts.has_key('r'))) |
|
2062 | 2083 | f = file(fname,'w') |
|
2063 | 2084 | f.write(cmds) |
|
2064 | 2085 | f.close() |
|
2065 | 2086 | print 'The following commands were written to file `%s`:' % fname |
|
2066 | 2087 | print cmds |
|
2067 | 2088 | |
|
2068 | 2089 | def _edit_macro(self,mname,macro): |
|
2069 | 2090 | """open an editor with the macro data in a file""" |
|
2070 | 2091 | filename = self.shell.mktempfile(macro.value) |
|
2071 | 2092 | self.shell.hooks.editor(filename) |
|
2072 | 2093 | |
|
2073 | 2094 | # and make a new macro object, to replace the old one |
|
2074 | 2095 | mfile = open(filename) |
|
2075 | 2096 | mvalue = mfile.read() |
|
2076 | 2097 | mfile.close() |
|
2077 | 2098 | self.shell.user_ns[mname] = Macro(mvalue) |
|
2078 | 2099 | |
|
2079 | 2100 | def magic_ed(self,parameter_s=''): |
|
2080 | 2101 | """Alias to %edit.""" |
|
2081 | 2102 | return self.magic_edit(parameter_s) |
|
2082 | 2103 | |
|
2083 | 2104 | @testdec.skip_doctest |
|
2084 | 2105 | def magic_edit(self,parameter_s='',last_call=['','']): |
|
2085 | 2106 | """Bring up an editor and execute the resulting code. |
|
2086 | 2107 | |
|
2087 | 2108 | Usage: |
|
2088 | 2109 | %edit [options] [args] |
|
2089 | 2110 | |
|
2090 | 2111 | %edit runs IPython's editor hook. The default version of this hook is |
|
2091 | 2112 | set to call the __IPYTHON__.rc.editor command. This is read from your |
|
2092 | 2113 | environment variable $EDITOR. If this isn't found, it will default to |
|
2093 | 2114 | vi under Linux/Unix and to notepad under Windows. See the end of this |
|
2094 | 2115 | docstring for how to change the editor hook. |
|
2095 | 2116 | |
|
2096 | 2117 | You can also set the value of this editor via the command line option |
|
2097 | 2118 | '-editor' or in your ipythonrc file. This is useful if you wish to use |
|
2098 | 2119 | specifically for IPython an editor different from your typical default |
|
2099 | 2120 | (and for Windows users who typically don't set environment variables). |
|
2100 | 2121 | |
|
2101 | 2122 | This command allows you to conveniently edit multi-line code right in |
|
2102 | 2123 | your IPython session. |
|
2103 | 2124 | |
|
2104 | 2125 | If called without arguments, %edit opens up an empty editor with a |
|
2105 | 2126 | temporary file and will execute the contents of this file when you |
|
2106 | 2127 | close it (don't forget to save it!). |
|
2107 | 2128 | |
|
2108 | 2129 | |
|
2109 | 2130 | Options: |
|
2110 | 2131 | |
|
2111 | 2132 | -n <number>: open the editor at a specified line number. By default, |
|
2112 | 2133 | the IPython editor hook uses the unix syntax 'editor +N filename', but |
|
2113 | 2134 | you can configure this by providing your own modified hook if your |
|
2114 | 2135 | favorite editor supports line-number specifications with a different |
|
2115 | 2136 | syntax. |
|
2116 | 2137 | |
|
2117 | 2138 | -p: this will call the editor with the same data as the previous time |
|
2118 | 2139 | it was used, regardless of how long ago (in your current session) it |
|
2119 | 2140 | was. |
|
2120 | 2141 | |
|
2121 | 2142 | -r: use 'raw' input. This option only applies to input taken from the |
|
2122 | 2143 | user's history. By default, the 'processed' history is used, so that |
|
2123 | 2144 | magics are loaded in their transformed version to valid Python. If |
|
2124 | 2145 | this option is given, the raw input as typed as the command line is |
|
2125 | 2146 | used instead. When you exit the editor, it will be executed by |
|
2126 | 2147 | IPython's own processor. |
|
2127 | 2148 | |
|
2128 | 2149 | -x: do not execute the edited code immediately upon exit. This is |
|
2129 | 2150 | mainly useful if you are editing programs which need to be called with |
|
2130 | 2151 | command line arguments, which you can then do using %run. |
|
2131 | 2152 | |
|
2132 | 2153 | |
|
2133 | 2154 | Arguments: |
|
2134 | 2155 | |
|
2135 | 2156 | If arguments are given, the following possibilites exist: |
|
2136 | 2157 | |
|
2137 | 2158 | - The arguments are numbers or pairs of colon-separated numbers (like |
|
2138 | 2159 | 1 4:8 9). These are interpreted as lines of previous input to be |
|
2139 | 2160 | loaded into the editor. The syntax is the same of the %macro command. |
|
2140 | 2161 | |
|
2141 | 2162 | - If the argument doesn't start with a number, it is evaluated as a |
|
2142 | 2163 | variable and its contents loaded into the editor. You can thus edit |
|
2143 | 2164 | any string which contains python code (including the result of |
|
2144 | 2165 | previous edits). |
|
2145 | 2166 | |
|
2146 | 2167 | - If the argument is the name of an object (other than a string), |
|
2147 | 2168 | IPython will try to locate the file where it was defined and open the |
|
2148 | 2169 | editor at the point where it is defined. You can use `%edit function` |
|
2149 | 2170 | to load an editor exactly at the point where 'function' is defined, |
|
2150 | 2171 | edit it and have the file be executed automatically. |
|
2151 | 2172 | |
|
2152 | 2173 | If the object is a macro (see %macro for details), this opens up your |
|
2153 | 2174 | specified editor with a temporary file containing the macro's data. |
|
2154 | 2175 | Upon exit, the macro is reloaded with the contents of the file. |
|
2155 | 2176 | |
|
2156 | 2177 | Note: opening at an exact line is only supported under Unix, and some |
|
2157 | 2178 | editors (like kedit and gedit up to Gnome 2.8) do not understand the |
|
2158 | 2179 | '+NUMBER' parameter necessary for this feature. Good editors like |
|
2159 | 2180 | (X)Emacs, vi, jed, pico and joe all do. |
|
2160 | 2181 | |
|
2161 | 2182 | - If the argument is not found as a variable, IPython will look for a |
|
2162 | 2183 | file with that name (adding .py if necessary) and load it into the |
|
2163 | 2184 | editor. It will execute its contents with execfile() when you exit, |
|
2164 | 2185 | loading any code in the file into your interactive namespace. |
|
2165 | 2186 | |
|
2166 | 2187 | After executing your code, %edit will return as output the code you |
|
2167 | 2188 | typed in the editor (except when it was an existing file). This way |
|
2168 | 2189 | you can reload the code in further invocations of %edit as a variable, |
|
2169 | 2190 | via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of |
|
2170 | 2191 | the output. |
|
2171 | 2192 | |
|
2172 | 2193 | Note that %edit is also available through the alias %ed. |
|
2173 | 2194 | |
|
2174 | 2195 | This is an example of creating a simple function inside the editor and |
|
2175 | 2196 | then modifying it. First, start up the editor: |
|
2176 | 2197 | |
|
2177 | 2198 | In [1]: ed |
|
2178 | 2199 | Editing... done. Executing edited code... |
|
2179 | 2200 | Out[1]: 'def foo():n print "foo() was defined in an editing session"n' |
|
2180 | 2201 | |
|
2181 | 2202 | We can then call the function foo(): |
|
2182 | 2203 | |
|
2183 | 2204 | In [2]: foo() |
|
2184 | 2205 | foo() was defined in an editing session |
|
2185 | 2206 | |
|
2186 | 2207 | Now we edit foo. IPython automatically loads the editor with the |
|
2187 | 2208 | (temporary) file where foo() was previously defined: |
|
2188 | 2209 | |
|
2189 | 2210 | In [3]: ed foo |
|
2190 | 2211 | Editing... done. Executing edited code... |
|
2191 | 2212 | |
|
2192 | 2213 | And if we call foo() again we get the modified version: |
|
2193 | 2214 | |
|
2194 | 2215 | In [4]: foo() |
|
2195 | 2216 | foo() has now been changed! |
|
2196 | 2217 | |
|
2197 | 2218 | Here is an example of how to edit a code snippet successive |
|
2198 | 2219 | times. First we call the editor: |
|
2199 | 2220 | |
|
2200 | 2221 | In [5]: ed |
|
2201 | 2222 | Editing... done. Executing edited code... |
|
2202 | 2223 | hello |
|
2203 | 2224 | Out[5]: "print 'hello'n" |
|
2204 | 2225 | |
|
2205 | 2226 | Now we call it again with the previous output (stored in _): |
|
2206 | 2227 | |
|
2207 | 2228 | In [6]: ed _ |
|
2208 | 2229 | Editing... done. Executing edited code... |
|
2209 | 2230 | hello world |
|
2210 | 2231 | Out[6]: "print 'hello world'n" |
|
2211 | 2232 | |
|
2212 | 2233 | Now we call it with the output #8 (stored in _8, also as Out[8]): |
|
2213 | 2234 | |
|
2214 | 2235 | In [7]: ed _8 |
|
2215 | 2236 | Editing... done. Executing edited code... |
|
2216 | 2237 | hello again |
|
2217 | 2238 | Out[7]: "print 'hello again'n" |
|
2218 | 2239 | |
|
2219 | 2240 | |
|
2220 | 2241 | Changing the default editor hook: |
|
2221 | 2242 | |
|
2222 | 2243 | If you wish to write your own editor hook, you can put it in a |
|
2223 | 2244 | configuration file which you load at startup time. The default hook |
|
2224 | 2245 | is defined in the IPython.hooks module, and you can use that as a |
|
2225 | 2246 | starting example for further modifications. That file also has |
|
2226 | 2247 | general instructions on how to set a new hook for use once you've |
|
2227 | 2248 | defined it.""" |
|
2228 | 2249 | |
|
2229 | 2250 | # FIXME: This function has become a convoluted mess. It needs a |
|
2230 | 2251 | # ground-up rewrite with clean, simple logic. |
|
2231 | 2252 | |
|
2232 | 2253 | def make_filename(arg): |
|
2233 | 2254 | "Make a filename from the given args" |
|
2234 | 2255 | try: |
|
2235 | 2256 | filename = get_py_filename(arg) |
|
2236 | 2257 | except IOError: |
|
2237 | 2258 | if args.endswith('.py'): |
|
2238 | 2259 | filename = arg |
|
2239 | 2260 | else: |
|
2240 | 2261 | filename = None |
|
2241 | 2262 | return filename |
|
2242 | 2263 | |
|
2243 | 2264 | # custom exceptions |
|
2244 | 2265 | class DataIsObject(Exception): pass |
|
2245 | 2266 | |
|
2246 | 2267 | opts,args = self.parse_options(parameter_s,'prxn:') |
|
2247 | 2268 | # Set a few locals from the options for convenience: |
|
2248 | 2269 | opts_p = opts.has_key('p') |
|
2249 | 2270 | opts_r = opts.has_key('r') |
|
2250 | 2271 | |
|
2251 | 2272 | # Default line number value |
|
2252 | 2273 | lineno = opts.get('n',None) |
|
2253 | 2274 | |
|
2254 | 2275 | if opts_p: |
|
2255 | 2276 | args = '_%s' % last_call[0] |
|
2256 | 2277 | if not self.shell.user_ns.has_key(args): |
|
2257 | 2278 | args = last_call[1] |
|
2258 | 2279 | |
|
2259 | 2280 | # use last_call to remember the state of the previous call, but don't |
|
2260 | 2281 | # let it be clobbered by successive '-p' calls. |
|
2261 | 2282 | try: |
|
2262 | 2283 | last_call[0] = self.shell.outputcache.prompt_count |
|
2263 | 2284 | if not opts_p: |
|
2264 | 2285 | last_call[1] = parameter_s |
|
2265 | 2286 | except: |
|
2266 | 2287 | pass |
|
2267 | 2288 | |
|
2268 | 2289 | # by default this is done with temp files, except when the given |
|
2269 | 2290 | # arg is a filename |
|
2270 | 2291 | use_temp = 1 |
|
2271 | 2292 | |
|
2272 | 2293 | if re.match(r'\d',args): |
|
2273 | 2294 | # Mode where user specifies ranges of lines, like in %macro. |
|
2274 | 2295 | # This means that you can't edit files whose names begin with |
|
2275 | 2296 | # numbers this way. Tough. |
|
2276 | 2297 | ranges = args.split() |
|
2277 | 2298 | data = ''.join(self.extract_input_slices(ranges,opts_r)) |
|
2278 | 2299 | elif args.endswith('.py'): |
|
2279 | 2300 | filename = make_filename(args) |
|
2280 | 2301 | data = '' |
|
2281 | 2302 | use_temp = 0 |
|
2282 | 2303 | elif args: |
|
2283 | 2304 | try: |
|
2284 | 2305 | # Load the parameter given as a variable. If not a string, |
|
2285 | 2306 | # process it as an object instead (below) |
|
2286 | 2307 | |
|
2287 | 2308 | #print '*** args',args,'type',type(args) # dbg |
|
2288 | 2309 | data = eval(args,self.shell.user_ns) |
|
2289 | 2310 | if not type(data) in StringTypes: |
|
2290 | 2311 | raise DataIsObject |
|
2291 | 2312 | |
|
2292 | 2313 | except (NameError,SyntaxError): |
|
2293 | 2314 | # given argument is not a variable, try as a filename |
|
2294 | 2315 | filename = make_filename(args) |
|
2295 | 2316 | if filename is None: |
|
2296 | 2317 | warn("Argument given (%s) can't be found as a variable " |
|
2297 | 2318 | "or as a filename." % args) |
|
2298 | 2319 | return |
|
2299 | 2320 | |
|
2300 | 2321 | data = '' |
|
2301 | 2322 | use_temp = 0 |
|
2302 | 2323 | except DataIsObject: |
|
2303 | 2324 | |
|
2304 | 2325 | # macros have a special edit function |
|
2305 | 2326 | if isinstance(data,Macro): |
|
2306 | 2327 | self._edit_macro(args,data) |
|
2307 | 2328 | return |
|
2308 | 2329 | |
|
2309 | 2330 | # For objects, try to edit the file where they are defined |
|
2310 | 2331 | try: |
|
2311 | 2332 | filename = inspect.getabsfile(data) |
|
2312 | 2333 | if 'fakemodule' in filename.lower() and inspect.isclass(data): |
|
2313 | 2334 | # class created by %edit? Try to find source |
|
2314 | 2335 | # by looking for method definitions instead, the |
|
2315 | 2336 | # __module__ in those classes is FakeModule. |
|
2316 | 2337 | attrs = [getattr(data, aname) for aname in dir(data)] |
|
2317 | 2338 | for attr in attrs: |
|
2318 | 2339 | if not inspect.ismethod(attr): |
|
2319 | 2340 | continue |
|
2320 | 2341 | filename = inspect.getabsfile(attr) |
|
2321 | 2342 | if filename and 'fakemodule' not in filename.lower(): |
|
2322 | 2343 | # change the attribute to be the edit target instead |
|
2323 | 2344 | data = attr |
|
2324 | 2345 | break |
|
2325 | 2346 | |
|
2326 | 2347 | datafile = 1 |
|
2327 | 2348 | except TypeError: |
|
2328 | 2349 | filename = make_filename(args) |
|
2329 | 2350 | datafile = 1 |
|
2330 | 2351 | warn('Could not find file where `%s` is defined.\n' |
|
2331 | 2352 | 'Opening a file named `%s`' % (args,filename)) |
|
2332 | 2353 | # Now, make sure we can actually read the source (if it was in |
|
2333 | 2354 | # a temp file it's gone by now). |
|
2334 | 2355 | if datafile: |
|
2335 | 2356 | try: |
|
2336 | 2357 | if lineno is None: |
|
2337 | 2358 | lineno = inspect.getsourcelines(data)[1] |
|
2338 | 2359 | except IOError: |
|
2339 | 2360 | filename = make_filename(args) |
|
2340 | 2361 | if filename is None: |
|
2341 | 2362 | warn('The file `%s` where `%s` was defined cannot ' |
|
2342 | 2363 | 'be read.' % (filename,data)) |
|
2343 | 2364 | return |
|
2344 | 2365 | use_temp = 0 |
|
2345 | 2366 | else: |
|
2346 | 2367 | data = '' |
|
2347 | 2368 | |
|
2348 | 2369 | if use_temp: |
|
2349 | 2370 | filename = self.shell.mktempfile(data) |
|
2350 | 2371 | print 'IPython will make a temporary file named:',filename |
|
2351 | 2372 | |
|
2352 | 2373 | # do actual editing here |
|
2353 | 2374 | print 'Editing...', |
|
2354 | 2375 | sys.stdout.flush() |
|
2355 | 2376 | try: |
|
2356 | 2377 | self.shell.hooks.editor(filename,lineno) |
|
2357 | 2378 | except IPython.ipapi.TryNext: |
|
2358 | 2379 | warn('Could not open editor') |
|
2359 | 2380 | return |
|
2360 | 2381 | |
|
2361 | 2382 | # XXX TODO: should this be generalized for all string vars? |
|
2362 | 2383 | # For now, this is special-cased to blocks created by cpaste |
|
2363 | 2384 | if args.strip() == 'pasted_block': |
|
2364 | 2385 | self.shell.user_ns['pasted_block'] = file_read(filename) |
|
2365 | 2386 | |
|
2366 | 2387 | if opts.has_key('x'): # -x prevents actual execution |
|
2367 | 2388 | |
|
2368 | 2389 | else: |
|
2369 | 2390 | print 'done. Executing edited code...' |
|
2370 | 2391 | if opts_r: |
|
2371 | 2392 | self.shell.runlines(file_read(filename)) |
|
2372 | 2393 | else: |
|
2373 | 2394 | self.shell.safe_execfile(filename,self.shell.user_ns, |
|
2374 | 2395 | self.shell.user_ns) |
|
2375 | 2396 | |
|
2376 | 2397 | |
|
2377 | 2398 | if use_temp: |
|
2378 | 2399 | try: |
|
2379 | 2400 | return open(filename).read() |
|
2380 | 2401 | except IOError,msg: |
|
2381 | 2402 | if msg.filename == filename: |
|
2382 | 2403 | warn('File not found. Did you forget to save?') |
|
2383 | 2404 | return |
|
2384 | 2405 | else: |
|
2385 | 2406 | self.shell.showtraceback() |
|
2386 | 2407 | |
|
2387 | 2408 | def magic_xmode(self,parameter_s = ''): |
|
2388 | 2409 | """Switch modes for the exception handlers. |
|
2389 | 2410 | |
|
2390 | 2411 | Valid modes: Plain, Context and Verbose. |
|
2391 | 2412 | |
|
2392 | 2413 | If called without arguments, acts as a toggle.""" |
|
2393 | 2414 | |
|
2394 | 2415 | def xmode_switch_err(name): |
|
2395 | 2416 | warn('Error changing %s exception modes.\n%s' % |
|
2396 | 2417 | (name,sys.exc_info()[1])) |
|
2397 | 2418 | |
|
2398 | 2419 | shell = self.shell |
|
2399 | 2420 | new_mode = parameter_s.strip().capitalize() |
|
2400 | 2421 | try: |
|
2401 | 2422 | shell.InteractiveTB.set_mode(mode=new_mode) |
|
2402 | 2423 | print 'Exception reporting mode:',shell.InteractiveTB.mode |
|
2403 | 2424 | except: |
|
2404 | 2425 | xmode_switch_err('user') |
|
2405 | 2426 | |
|
2406 | 2427 | # threaded shells use a special handler in sys.excepthook |
|
2407 | 2428 | if shell.isthreaded: |
|
2408 | 2429 | try: |
|
2409 | 2430 | shell.sys_excepthook.set_mode(mode=new_mode) |
|
2410 | 2431 | except: |
|
2411 | 2432 | xmode_switch_err('threaded') |
|
2412 | 2433 | |
|
2413 | 2434 | def magic_colors(self,parameter_s = ''): |
|
2414 | 2435 | """Switch color scheme for prompts, info system and exception handlers. |
|
2415 | 2436 | |
|
2416 | 2437 | Currently implemented schemes: NoColor, Linux, LightBG. |
|
2417 | 2438 | |
|
2418 | 2439 | Color scheme names are not case-sensitive.""" |
|
2419 | 2440 | |
|
2420 | 2441 | def color_switch_err(name): |
|
2421 | 2442 | warn('Error changing %s color schemes.\n%s' % |
|
2422 | 2443 | (name,sys.exc_info()[1])) |
|
2423 | 2444 | |
|
2424 | 2445 | |
|
2425 | 2446 | new_scheme = parameter_s.strip() |
|
2426 | 2447 | if not new_scheme: |
|
2427 | 2448 | raise UsageError( |
|
2428 | 2449 | "%colors: you must specify a color scheme. See '%colors?'") |
|
2429 | 2450 | return |
|
2430 | 2451 | # local shortcut |
|
2431 | 2452 | shell = self.shell |
|
2432 | 2453 | |
|
2433 | 2454 | import IPython.rlineimpl as readline |
|
2434 | 2455 | |
|
2435 | 2456 | if not readline.have_readline and sys.platform == "win32": |
|
2436 | 2457 | msg = """\ |
|
2437 | 2458 | Proper color support under MS Windows requires the pyreadline library. |
|
2438 | 2459 | You can find it at: |
|
2439 | 2460 | http://ipython.scipy.org/moin/PyReadline/Intro |
|
2440 | 2461 | Gary's readline needs the ctypes module, from: |
|
2441 | 2462 | http://starship.python.net/crew/theller/ctypes |
|
2442 | 2463 | (Note that ctypes is already part of Python versions 2.5 and newer). |
|
2443 | 2464 | |
|
2444 | 2465 | Defaulting color scheme to 'NoColor'""" |
|
2445 | 2466 | new_scheme = 'NoColor' |
|
2446 | 2467 | warn(msg) |
|
2447 | 2468 | |
|
2448 | 2469 | # readline option is 0 |
|
2449 | 2470 | if not shell.has_readline: |
|
2450 | 2471 | new_scheme = 'NoColor' |
|
2451 | 2472 | |
|
2452 | 2473 | # Set prompt colors |
|
2453 | 2474 | try: |
|
2454 | 2475 | shell.outputcache.set_colors(new_scheme) |
|
2455 | 2476 | except: |
|
2456 | 2477 | color_switch_err('prompt') |
|
2457 | 2478 | else: |
|
2458 | 2479 | shell.rc.colors = \ |
|
2459 | 2480 | shell.outputcache.color_table.active_scheme_name |
|
2460 | 2481 | # Set exception colors |
|
2461 | 2482 | try: |
|
2462 | 2483 | shell.InteractiveTB.set_colors(scheme = new_scheme) |
|
2463 | 2484 | shell.SyntaxTB.set_colors(scheme = new_scheme) |
|
2464 | 2485 | except: |
|
2465 | 2486 | color_switch_err('exception') |
|
2466 | 2487 | |
|
2467 | 2488 | # threaded shells use a verbose traceback in sys.excepthook |
|
2468 | 2489 | if shell.isthreaded: |
|
2469 | 2490 | try: |
|
2470 | 2491 | shell.sys_excepthook.set_colors(scheme=new_scheme) |
|
2471 | 2492 | except: |
|
2472 | 2493 | color_switch_err('system exception handler') |
|
2473 | 2494 | |
|
2474 | 2495 | # Set info (for 'object?') colors |
|
2475 | 2496 | if shell.rc.color_info: |
|
2476 | 2497 | try: |
|
2477 | 2498 | shell.inspector.set_active_scheme(new_scheme) |
|
2478 | 2499 | except: |
|
2479 | 2500 | color_switch_err('object inspector') |
|
2480 | 2501 | else: |
|
2481 | 2502 | shell.inspector.set_active_scheme('NoColor') |
|
2482 | 2503 | |
|
2483 | 2504 | def magic_color_info(self,parameter_s = ''): |
|
2484 | 2505 | """Toggle color_info. |
|
2485 | 2506 | |
|
2486 | 2507 | The color_info configuration parameter controls whether colors are |
|
2487 | 2508 | used for displaying object details (by things like %psource, %pfile or |
|
2488 | 2509 | the '?' system). This function toggles this value with each call. |
|
2489 | 2510 | |
|
2490 | 2511 | Note that unless you have a fairly recent pager (less works better |
|
2491 | 2512 | than more) in your system, using colored object information displays |
|
2492 | 2513 | will not work properly. Test it and see.""" |
|
2493 | 2514 | |
|
2494 | 2515 | self.shell.rc.color_info = 1 - self.shell.rc.color_info |
|
2495 | 2516 | self.magic_colors(self.shell.rc.colors) |
|
2496 | 2517 | print 'Object introspection functions have now coloring:', |
|
2497 | 2518 | print ['OFF','ON'][self.shell.rc.color_info] |
|
2498 | 2519 | |
|
2499 | 2520 | def magic_Pprint(self, parameter_s=''): |
|
2500 | 2521 | """Toggle pretty printing on/off.""" |
|
2501 | 2522 | |
|
2502 | 2523 | self.shell.rc.pprint = 1 - self.shell.rc.pprint |
|
2503 | 2524 | print 'Pretty printing has been turned', \ |
|
2504 | 2525 | ['OFF','ON'][self.shell.rc.pprint] |
|
2505 | 2526 | |
|
2506 | 2527 | def magic_exit(self, parameter_s=''): |
|
2507 | 2528 | """Exit IPython, confirming if configured to do so. |
|
2508 | 2529 | |
|
2509 | 2530 | You can configure whether IPython asks for confirmation upon exit by |
|
2510 | 2531 | setting the confirm_exit flag in the ipythonrc file.""" |
|
2511 | 2532 | |
|
2512 | 2533 | self.shell.exit() |
|
2513 | 2534 | |
|
2514 | 2535 | def magic_quit(self, parameter_s=''): |
|
2515 | 2536 | """Exit IPython, confirming if configured to do so (like %exit)""" |
|
2516 | 2537 | |
|
2517 | 2538 | self.shell.exit() |
|
2518 | 2539 | |
|
2519 | 2540 | def magic_Exit(self, parameter_s=''): |
|
2520 | 2541 | """Exit IPython without confirmation.""" |
|
2521 | 2542 | |
|
2522 | 2543 | self.shell.ask_exit() |
|
2523 | 2544 | |
|
2524 | 2545 | #...................................................................... |
|
2525 | 2546 | # Functions to implement unix shell-type things |
|
2526 | 2547 | |
|
2527 | 2548 | @testdec.skip_doctest |
|
2528 | 2549 | def magic_alias(self, parameter_s = ''): |
|
2529 | 2550 | """Define an alias for a system command. |
|
2530 | 2551 | |
|
2531 | 2552 | '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd' |
|
2532 | 2553 | |
|
2533 | 2554 | Then, typing 'alias_name params' will execute the system command 'cmd |
|
2534 | 2555 | params' (from your underlying operating system). |
|
2535 | 2556 | |
|
2536 | 2557 | Aliases have lower precedence than magic functions and Python normal |
|
2537 | 2558 | variables, so if 'foo' is both a Python variable and an alias, the |
|
2538 | 2559 | alias can not be executed until 'del foo' removes the Python variable. |
|
2539 | 2560 | |
|
2540 | 2561 | You can use the %l specifier in an alias definition to represent the |
|
2541 | 2562 | whole line when the alias is called. For example: |
|
2542 | 2563 | |
|
2543 | 2564 | In [2]: alias all echo "Input in brackets: <%l>" |
|
2544 | 2565 | In [3]: all hello world |
|
2545 | 2566 | Input in brackets: <hello world> |
|
2546 | 2567 | |
|
2547 | 2568 | You can also define aliases with parameters using %s specifiers (one |
|
2548 | 2569 | per parameter): |
|
2549 | 2570 | |
|
2550 | 2571 | In [1]: alias parts echo first %s second %s |
|
2551 | 2572 | In [2]: %parts A B |
|
2552 | 2573 | first A second B |
|
2553 | 2574 | In [3]: %parts A |
|
2554 | 2575 | Incorrect number of arguments: 2 expected. |
|
2555 | 2576 | parts is an alias to: 'echo first %s second %s' |
|
2556 | 2577 | |
|
2557 | 2578 | Note that %l and %s are mutually exclusive. You can only use one or |
|
2558 | 2579 | the other in your aliases. |
|
2559 | 2580 | |
|
2560 | 2581 | Aliases expand Python variables just like system calls using ! or !! |
|
2561 | 2582 | do: all expressions prefixed with '$' get expanded. For details of |
|
2562 | 2583 | the semantic rules, see PEP-215: |
|
2563 | 2584 | http://www.python.org/peps/pep-0215.html. This is the library used by |
|
2564 | 2585 | IPython for variable expansion. If you want to access a true shell |
|
2565 | 2586 | variable, an extra $ is necessary to prevent its expansion by IPython: |
|
2566 | 2587 | |
|
2567 | 2588 | In [6]: alias show echo |
|
2568 | 2589 | In [7]: PATH='A Python string' |
|
2569 | 2590 | In [8]: show $PATH |
|
2570 | 2591 | A Python string |
|
2571 | 2592 | In [9]: show $$PATH |
|
2572 | 2593 | /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:... |
|
2573 | 2594 | |
|
2574 | 2595 | You can use the alias facility to acess all of $PATH. See the %rehash |
|
2575 | 2596 | and %rehashx functions, which automatically create aliases for the |
|
2576 | 2597 | contents of your $PATH. |
|
2577 | 2598 | |
|
2578 | 2599 | If called with no parameters, %alias prints the current alias table.""" |
|
2579 | 2600 | |
|
2580 | 2601 | par = parameter_s.strip() |
|
2581 | 2602 | if not par: |
|
2582 | 2603 | stored = self.db.get('stored_aliases', {} ) |
|
2583 | 2604 | atab = self.shell.alias_table |
|
2584 | 2605 | aliases = atab.keys() |
|
2585 | 2606 | aliases.sort() |
|
2586 | 2607 | res = [] |
|
2587 | 2608 | showlast = [] |
|
2588 | 2609 | for alias in aliases: |
|
2589 | 2610 | special = False |
|
2590 | 2611 | try: |
|
2591 | 2612 | tgt = atab[alias][1] |
|
2592 | 2613 | except (TypeError, AttributeError): |
|
2593 | 2614 | # unsubscriptable? probably a callable |
|
2594 | 2615 | tgt = atab[alias] |
|
2595 | 2616 | special = True |
|
2596 | 2617 | # 'interesting' aliases |
|
2597 | 2618 | if (alias in stored or |
|
2598 | 2619 | special or |
|
2599 | 2620 | alias.lower() != os.path.splitext(tgt)[0].lower() or |
|
2600 | 2621 | ' ' in tgt): |
|
2601 | 2622 | showlast.append((alias, tgt)) |
|
2602 | 2623 | else: |
|
2603 | 2624 | res.append((alias, tgt )) |
|
2604 | 2625 | |
|
2605 | 2626 | # show most interesting aliases last |
|
2606 | 2627 | res.extend(showlast) |
|
2607 | 2628 | print "Total number of aliases:",len(aliases) |
|
2608 | 2629 | return res |
|
2609 | 2630 | try: |
|
2610 | 2631 | alias,cmd = par.split(None,1) |
|
2611 | 2632 | except: |
|
2612 | 2633 | print OInspect.getdoc(self.magic_alias) |
|
2613 | 2634 | else: |
|
2614 | 2635 | nargs = cmd.count('%s') |
|
2615 | 2636 | if nargs>0 and cmd.find('%l')>=0: |
|
2616 | 2637 | error('The %s and %l specifiers are mutually exclusive ' |
|
2617 | 2638 | 'in alias definitions.') |
|
2618 | 2639 | else: # all looks OK |
|
2619 | 2640 | self.shell.alias_table[alias] = (nargs,cmd) |
|
2620 | 2641 | self.shell.alias_table_validate(verbose=0) |
|
2621 | 2642 | # end magic_alias |
|
2622 | 2643 | |
|
2623 | 2644 | def magic_unalias(self, parameter_s = ''): |
|
2624 | 2645 | """Remove an alias""" |
|
2625 | 2646 | |
|
2626 | 2647 | aname = parameter_s.strip() |
|
2627 | 2648 | if aname in self.shell.alias_table: |
|
2628 | 2649 | del self.shell.alias_table[aname] |
|
2629 | 2650 | stored = self.db.get('stored_aliases', {} ) |
|
2630 | 2651 | if aname in stored: |
|
2631 | 2652 | print "Removing %stored alias",aname |
|
2632 | 2653 | del stored[aname] |
|
2633 | 2654 | self.db['stored_aliases'] = stored |
|
2634 | 2655 | |
|
2635 | 2656 | |
|
2636 | 2657 | def magic_rehashx(self, parameter_s = ''): |
|
2637 | 2658 | """Update the alias table with all executable files in $PATH. |
|
2638 | 2659 | |
|
2639 | 2660 | This version explicitly checks that every entry in $PATH is a file |
|
2640 | 2661 | with execute access (os.X_OK), so it is much slower than %rehash. |
|
2641 | 2662 | |
|
2642 | 2663 | Under Windows, it checks executability as a match agains a |
|
2643 | 2664 | '|'-separated string of extensions, stored in the IPython config |
|
2644 | 2665 | variable win_exec_ext. This defaults to 'exe|com|bat'. |
|
2645 | 2666 | |
|
2646 | 2667 | This function also resets the root module cache of module completer, |
|
2647 | 2668 | used on slow filesystems. |
|
2648 | 2669 | """ |
|
2649 | 2670 | |
|
2650 | 2671 | |
|
2651 | 2672 | ip = self.api |
|
2652 | 2673 | |
|
2653 | 2674 | # for the benefit of module completer in ipy_completers.py |
|
2654 | 2675 | del ip.db['rootmodules'] |
|
2655 | 2676 | |
|
2656 | 2677 | path = [os.path.abspath(os.path.expanduser(p)) for p in |
|
2657 | 2678 | os.environ.get('PATH','').split(os.pathsep)] |
|
2658 | 2679 | path = filter(os.path.isdir,path) |
|
2659 | 2680 | |
|
2660 | 2681 | alias_table = self.shell.alias_table |
|
2661 | 2682 | syscmdlist = [] |
|
2662 | 2683 | if os.name == 'posix': |
|
2663 | 2684 | isexec = lambda fname:os.path.isfile(fname) and \ |
|
2664 | 2685 | os.access(fname,os.X_OK) |
|
2665 | 2686 | else: |
|
2666 | 2687 | |
|
2667 | 2688 | try: |
|
2668 | 2689 | winext = os.environ['pathext'].replace(';','|').replace('.','') |
|
2669 | 2690 | except KeyError: |
|
2670 | 2691 | winext = 'exe|com|bat|py' |
|
2671 | 2692 | if 'py' not in winext: |
|
2672 | 2693 | winext += '|py' |
|
2673 | 2694 | execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE) |
|
2674 | 2695 | isexec = lambda fname:os.path.isfile(fname) and execre.match(fname) |
|
2675 | 2696 | savedir = os.getcwd() |
|
2676 | 2697 | try: |
|
2677 | 2698 | # write the whole loop for posix/Windows so we don't have an if in |
|
2678 | 2699 | # the innermost part |
|
2679 | 2700 | if os.name == 'posix': |
|
2680 | 2701 | for pdir in path: |
|
2681 | 2702 | os.chdir(pdir) |
|
2682 | 2703 | for ff in os.listdir(pdir): |
|
2683 | 2704 | if isexec(ff) and ff not in self.shell.no_alias: |
|
2684 | 2705 | # each entry in the alias table must be (N,name), |
|
2685 | 2706 | # where N is the number of positional arguments of the |
|
2686 | 2707 | # alias. |
|
2687 | 2708 | # Dots will be removed from alias names, since ipython |
|
2688 | 2709 | # assumes names with dots to be python code |
|
2689 | 2710 | alias_table[ff.replace('.','')] = (0,ff) |
|
2690 | 2711 | syscmdlist.append(ff) |
|
2691 | 2712 | else: |
|
2692 | 2713 | for pdir in path: |
|
2693 | 2714 | os.chdir(pdir) |
|
2694 | 2715 | for ff in os.listdir(pdir): |
|
2695 | 2716 | base, ext = os.path.splitext(ff) |
|
2696 | 2717 | if isexec(ff) and base.lower() not in self.shell.no_alias: |
|
2697 | 2718 | if ext.lower() == '.exe': |
|
2698 | 2719 | ff = base |
|
2699 | 2720 | alias_table[base.lower().replace('.','')] = (0,ff) |
|
2700 | 2721 | syscmdlist.append(ff) |
|
2701 | 2722 | # Make sure the alias table doesn't contain keywords or builtins |
|
2702 | 2723 | self.shell.alias_table_validate() |
|
2703 | 2724 | # Call again init_auto_alias() so we get 'rm -i' and other |
|
2704 | 2725 | # modified aliases since %rehashx will probably clobber them |
|
2705 | 2726 | |
|
2706 | 2727 | # no, we don't want them. if %rehashx clobbers them, good, |
|
2707 | 2728 | # we'll probably get better versions |
|
2708 | 2729 | # self.shell.init_auto_alias() |
|
2709 | 2730 | db = ip.db |
|
2710 | 2731 | db['syscmdlist'] = syscmdlist |
|
2711 | 2732 | finally: |
|
2712 | 2733 | os.chdir(savedir) |
|
2713 | 2734 | |
|
2714 | 2735 | def magic_pwd(self, parameter_s = ''): |
|
2715 | 2736 | """Return the current working directory path.""" |
|
2716 | 2737 | return os.getcwd() |
|
2717 | 2738 | |
|
2718 | 2739 | def magic_cd(self, parameter_s=''): |
|
2719 | 2740 | """Change the current working directory. |
|
2720 | 2741 | |
|
2721 | 2742 | This command automatically maintains an internal list of directories |
|
2722 | 2743 | you visit during your IPython session, in the variable _dh. The |
|
2723 | 2744 | command %dhist shows this history nicely formatted. You can also |
|
2724 | 2745 | do 'cd -<tab>' to see directory history conveniently. |
|
2725 | 2746 | |
|
2726 | 2747 | Usage: |
|
2727 | 2748 | |
|
2728 | 2749 | cd 'dir': changes to directory 'dir'. |
|
2729 | 2750 | |
|
2730 | 2751 | cd -: changes to the last visited directory. |
|
2731 | 2752 | |
|
2732 | 2753 | cd -<n>: changes to the n-th directory in the directory history. |
|
2733 | 2754 | |
|
2734 | 2755 | cd --foo: change to directory that matches 'foo' in history |
|
2735 | 2756 | |
|
2736 | 2757 | cd -b <bookmark_name>: jump to a bookmark set by %bookmark |
|
2737 | 2758 | (note: cd <bookmark_name> is enough if there is no |
|
2738 | 2759 | directory <bookmark_name>, but a bookmark with the name exists.) |
|
2739 | 2760 | 'cd -b <tab>' allows you to tab-complete bookmark names. |
|
2740 | 2761 | |
|
2741 | 2762 | Options: |
|
2742 | 2763 | |
|
2743 | 2764 | -q: quiet. Do not print the working directory after the cd command is |
|
2744 | 2765 | executed. By default IPython's cd command does print this directory, |
|
2745 | 2766 | since the default prompts do not display path information. |
|
2746 | 2767 | |
|
2747 | 2768 | Note that !cd doesn't work for this purpose because the shell where |
|
2748 | 2769 | !command runs is immediately discarded after executing 'command'.""" |
|
2749 | 2770 | |
|
2750 | 2771 | parameter_s = parameter_s.strip() |
|
2751 | 2772 | #bkms = self.shell.persist.get("bookmarks",{}) |
|
2752 | 2773 | |
|
2753 | 2774 | oldcwd = os.getcwd() |
|
2754 | 2775 | numcd = re.match(r'(-)(\d+)$',parameter_s) |
|
2755 | 2776 | # jump in directory history by number |
|
2756 | 2777 | if numcd: |
|
2757 | 2778 | nn = int(numcd.group(2)) |
|
2758 | 2779 | try: |
|
2759 | 2780 | ps = self.shell.user_ns['_dh'][nn] |
|
2760 | 2781 | except IndexError: |
|
2761 | 2782 | print 'The requested directory does not exist in history.' |
|
2762 | 2783 | return |
|
2763 | 2784 | else: |
|
2764 | 2785 | opts = {} |
|
2765 | 2786 | elif parameter_s.startswith('--'): |
|
2766 | 2787 | ps = None |
|
2767 | 2788 | fallback = None |
|
2768 | 2789 | pat = parameter_s[2:] |
|
2769 | 2790 | dh = self.shell.user_ns['_dh'] |
|
2770 | 2791 | # first search only by basename (last component) |
|
2771 | 2792 | for ent in reversed(dh): |
|
2772 | 2793 | if pat in os.path.basename(ent) and os.path.isdir(ent): |
|
2773 | 2794 | ps = ent |
|
2774 | 2795 | break |
|
2775 | 2796 | |
|
2776 | 2797 | if fallback is None and pat in ent and os.path.isdir(ent): |
|
2777 | 2798 | fallback = ent |
|
2778 | 2799 | |
|
2779 | 2800 | # if we have no last part match, pick the first full path match |
|
2780 | 2801 | if ps is None: |
|
2781 | 2802 | ps = fallback |
|
2782 | 2803 | |
|
2783 | 2804 | if ps is None: |
|
2784 | 2805 | print "No matching entry in directory history" |
|
2785 | 2806 | return |
|
2786 | 2807 | else: |
|
2787 | 2808 | opts = {} |
|
2788 | 2809 | |
|
2789 | 2810 | |
|
2790 | 2811 | else: |
|
2791 | 2812 | #turn all non-space-escaping backslashes to slashes, |
|
2792 | 2813 | # for c:\windows\directory\names\ |
|
2793 | 2814 | parameter_s = re.sub(r'\\(?! )','/', parameter_s) |
|
2794 | 2815 | opts,ps = self.parse_options(parameter_s,'qb',mode='string') |
|
2795 | 2816 | # jump to previous |
|
2796 | 2817 | if ps == '-': |
|
2797 | 2818 | try: |
|
2798 | 2819 | ps = self.shell.user_ns['_dh'][-2] |
|
2799 | 2820 | except IndexError: |
|
2800 | 2821 | raise UsageError('%cd -: No previous directory to change to.') |
|
2801 | 2822 | # jump to bookmark if needed |
|
2802 | 2823 | else: |
|
2803 | 2824 | if not os.path.isdir(ps) or opts.has_key('b'): |
|
2804 | 2825 | bkms = self.db.get('bookmarks', {}) |
|
2805 | 2826 | |
|
2806 | 2827 | if bkms.has_key(ps): |
|
2807 | 2828 | target = bkms[ps] |
|
2808 | 2829 | print '(bookmark:%s) -> %s' % (ps,target) |
|
2809 | 2830 | ps = target |
|
2810 | 2831 | else: |
|
2811 | 2832 | if opts.has_key('b'): |
|
2812 | 2833 | raise UsageError("Bookmark '%s' not found. " |
|
2813 | 2834 | "Use '%%bookmark -l' to see your bookmarks." % ps) |
|
2814 | 2835 | |
|
2815 | 2836 | # at this point ps should point to the target dir |
|
2816 | 2837 | if ps: |
|
2817 | 2838 | try: |
|
2818 | 2839 | os.chdir(os.path.expanduser(ps)) |
|
2819 | 2840 | if self.shell.rc.term_title: |
|
2820 | 2841 | #print 'set term title:',self.shell.rc.term_title # dbg |
|
2821 | 2842 | platutils.set_term_title('IPy ' + abbrev_cwd()) |
|
2822 | 2843 | except OSError: |
|
2823 | 2844 | print sys.exc_info()[1] |
|
2824 | 2845 | else: |
|
2825 | 2846 | cwd = os.getcwd() |
|
2826 | 2847 | dhist = self.shell.user_ns['_dh'] |
|
2827 | 2848 | if oldcwd != cwd: |
|
2828 | 2849 | dhist.append(cwd) |
|
2829 | 2850 | self.db['dhist'] = compress_dhist(dhist)[-100:] |
|
2830 | 2851 | |
|
2831 | 2852 | else: |
|
2832 | 2853 | os.chdir(self.shell.home_dir) |
|
2833 | 2854 | if self.shell.rc.term_title: |
|
2834 | 2855 | platutils.set_term_title("IPy ~") |
|
2835 | 2856 | cwd = os.getcwd() |
|
2836 | 2857 | dhist = self.shell.user_ns['_dh'] |
|
2837 | 2858 | |
|
2838 | 2859 | if oldcwd != cwd: |
|
2839 | 2860 | dhist.append(cwd) |
|
2840 | 2861 | self.db['dhist'] = compress_dhist(dhist)[-100:] |
|
2841 | 2862 | if not 'q' in opts and self.shell.user_ns['_dh']: |
|
2842 | 2863 | print self.shell.user_ns['_dh'][-1] |
|
2843 | 2864 | |
|
2844 | 2865 | |
|
2845 | 2866 | def magic_env(self, parameter_s=''): |
|
2846 | 2867 | """List environment variables.""" |
|
2847 | 2868 | |
|
2848 | 2869 | return os.environ.data |
|
2849 | 2870 | |
|
2850 | 2871 | def magic_pushd(self, parameter_s=''): |
|
2851 | 2872 | """Place the current dir on stack and change directory. |
|
2852 | 2873 | |
|
2853 | 2874 | Usage:\\ |
|
2854 | 2875 | %pushd ['dirname'] |
|
2855 | 2876 | """ |
|
2856 | 2877 | |
|
2857 | 2878 | dir_s = self.shell.dir_stack |
|
2858 | 2879 | tgt = os.path.expanduser(parameter_s) |
|
2859 | 2880 | cwd = os.getcwd().replace(self.home_dir,'~') |
|
2860 | 2881 | if tgt: |
|
2861 | 2882 | self.magic_cd(parameter_s) |
|
2862 | 2883 | dir_s.insert(0,cwd) |
|
2863 | 2884 | return self.magic_dirs() |
|
2864 | 2885 | |
|
2865 | 2886 | def magic_popd(self, parameter_s=''): |
|
2866 | 2887 | """Change to directory popped off the top of the stack. |
|
2867 | 2888 | """ |
|
2868 | 2889 | if not self.shell.dir_stack: |
|
2869 | 2890 | raise UsageError("%popd on empty stack") |
|
2870 | 2891 | top = self.shell.dir_stack.pop(0) |
|
2871 | 2892 | self.magic_cd(top) |
|
2872 | 2893 | print "popd ->",top |
|
2873 | 2894 | |
|
2874 | 2895 | def magic_dirs(self, parameter_s=''): |
|
2875 | 2896 | """Return the current directory stack.""" |
|
2876 | 2897 | |
|
2877 | 2898 | return self.shell.dir_stack |
|
2878 | 2899 | |
|
2879 | 2900 | def magic_dhist(self, parameter_s=''): |
|
2880 | 2901 | """Print your history of visited directories. |
|
2881 | 2902 | |
|
2882 | 2903 | %dhist -> print full history\\ |
|
2883 | 2904 | %dhist n -> print last n entries only\\ |
|
2884 | 2905 | %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\ |
|
2885 | 2906 | |
|
2886 | 2907 | This history is automatically maintained by the %cd command, and |
|
2887 | 2908 | always available as the global list variable _dh. You can use %cd -<n> |
|
2888 | 2909 | to go to directory number <n>. |
|
2889 | 2910 | |
|
2890 | 2911 | Note that most of time, you should view directory history by entering |
|
2891 | 2912 | cd -<TAB>. |
|
2892 | 2913 | |
|
2893 | 2914 | """ |
|
2894 | 2915 | |
|
2895 | 2916 | dh = self.shell.user_ns['_dh'] |
|
2896 | 2917 | if parameter_s: |
|
2897 | 2918 | try: |
|
2898 | 2919 | args = map(int,parameter_s.split()) |
|
2899 | 2920 | except: |
|
2900 | 2921 | self.arg_err(Magic.magic_dhist) |
|
2901 | 2922 | return |
|
2902 | 2923 | if len(args) == 1: |
|
2903 | 2924 | ini,fin = max(len(dh)-(args[0]),0),len(dh) |
|
2904 | 2925 | elif len(args) == 2: |
|
2905 | 2926 | ini,fin = args |
|
2906 | 2927 | else: |
|
2907 | 2928 | self.arg_err(Magic.magic_dhist) |
|
2908 | 2929 | return |
|
2909 | 2930 | else: |
|
2910 | 2931 | ini,fin = 0,len(dh) |
|
2911 | 2932 | nlprint(dh, |
|
2912 | 2933 | header = 'Directory history (kept in _dh)', |
|
2913 | 2934 | start=ini,stop=fin) |
|
2914 | 2935 | |
|
2915 | 2936 | @testdec.skip_doctest |
|
2916 | 2937 | def magic_sc(self, parameter_s=''): |
|
2917 | 2938 | """Shell capture - execute a shell command and capture its output. |
|
2918 | 2939 | |
|
2919 | 2940 | DEPRECATED. Suboptimal, retained for backwards compatibility. |
|
2920 | 2941 | |
|
2921 | 2942 | You should use the form 'var = !command' instead. Example: |
|
2922 | 2943 | |
|
2923 | 2944 | "%sc -l myfiles = ls ~" should now be written as |
|
2924 | 2945 | |
|
2925 | 2946 | "myfiles = !ls ~" |
|
2926 | 2947 | |
|
2927 | 2948 | myfiles.s, myfiles.l and myfiles.n still apply as documented |
|
2928 | 2949 | below. |
|
2929 | 2950 | |
|
2930 | 2951 | -- |
|
2931 | 2952 | %sc [options] varname=command |
|
2932 | 2953 | |
|
2933 | 2954 | IPython will run the given command using commands.getoutput(), and |
|
2934 | 2955 | will then update the user's interactive namespace with a variable |
|
2935 | 2956 | called varname, containing the value of the call. Your command can |
|
2936 | 2957 | contain shell wildcards, pipes, etc. |
|
2937 | 2958 | |
|
2938 | 2959 | The '=' sign in the syntax is mandatory, and the variable name you |
|
2939 | 2960 | supply must follow Python's standard conventions for valid names. |
|
2940 | 2961 | |
|
2941 | 2962 | (A special format without variable name exists for internal use) |
|
2942 | 2963 | |
|
2943 | 2964 | Options: |
|
2944 | 2965 | |
|
2945 | 2966 | -l: list output. Split the output on newlines into a list before |
|
2946 | 2967 | assigning it to the given variable. By default the output is stored |
|
2947 | 2968 | as a single string. |
|
2948 | 2969 | |
|
2949 | 2970 | -v: verbose. Print the contents of the variable. |
|
2950 | 2971 | |
|
2951 | 2972 | In most cases you should not need to split as a list, because the |
|
2952 | 2973 | returned value is a special type of string which can automatically |
|
2953 | 2974 | provide its contents either as a list (split on newlines) or as a |
|
2954 | 2975 | space-separated string. These are convenient, respectively, either |
|
2955 | 2976 | for sequential processing or to be passed to a shell command. |
|
2956 | 2977 | |
|
2957 | 2978 | For example: |
|
2958 | 2979 | |
|
2959 | 2980 | # all-random |
|
2960 | 2981 | |
|
2961 | 2982 | # Capture into variable a |
|
2962 | 2983 | In [1]: sc a=ls *py |
|
2963 | 2984 | |
|
2964 | 2985 | # a is a string with embedded newlines |
|
2965 | 2986 | In [2]: a |
|
2966 | 2987 | Out[2]: 'setup.py\\nwin32_manual_post_install.py' |
|
2967 | 2988 | |
|
2968 | 2989 | # which can be seen as a list: |
|
2969 | 2990 | In [3]: a.l |
|
2970 | 2991 | Out[3]: ['setup.py', 'win32_manual_post_install.py'] |
|
2971 | 2992 | |
|
2972 | 2993 | # or as a whitespace-separated string: |
|
2973 | 2994 | In [4]: a.s |
|
2974 | 2995 | Out[4]: 'setup.py win32_manual_post_install.py' |
|
2975 | 2996 | |
|
2976 | 2997 | # a.s is useful to pass as a single command line: |
|
2977 | 2998 | In [5]: !wc -l $a.s |
|
2978 | 2999 | 146 setup.py |
|
2979 | 3000 | 130 win32_manual_post_install.py |
|
2980 | 3001 | 276 total |
|
2981 | 3002 | |
|
2982 | 3003 | # while the list form is useful to loop over: |
|
2983 | 3004 | In [6]: for f in a.l: |
|
2984 | 3005 | ...: !wc -l $f |
|
2985 | 3006 | ...: |
|
2986 | 3007 | 146 setup.py |
|
2987 | 3008 | 130 win32_manual_post_install.py |
|
2988 | 3009 | |
|
2989 | 3010 | Similiarly, the lists returned by the -l option are also special, in |
|
2990 | 3011 | the sense that you can equally invoke the .s attribute on them to |
|
2991 | 3012 | automatically get a whitespace-separated string from their contents: |
|
2992 | 3013 | |
|
2993 | 3014 | In [7]: sc -l b=ls *py |
|
2994 | 3015 | |
|
2995 | 3016 | In [8]: b |
|
2996 | 3017 | Out[8]: ['setup.py', 'win32_manual_post_install.py'] |
|
2997 | 3018 | |
|
2998 | 3019 | In [9]: b.s |
|
2999 | 3020 | Out[9]: 'setup.py win32_manual_post_install.py' |
|
3000 | 3021 | |
|
3001 | 3022 | In summary, both the lists and strings used for ouptut capture have |
|
3002 | 3023 | the following special attributes: |
|
3003 | 3024 | |
|
3004 | 3025 | .l (or .list) : value as list. |
|
3005 | 3026 | .n (or .nlstr): value as newline-separated string. |
|
3006 | 3027 | .s (or .spstr): value as space-separated string. |
|
3007 | 3028 | """ |
|
3008 | 3029 | |
|
3009 | 3030 | opts,args = self.parse_options(parameter_s,'lv') |
|
3010 | 3031 | # Try to get a variable name and command to run |
|
3011 | 3032 | try: |
|
3012 | 3033 | # the variable name must be obtained from the parse_options |
|
3013 | 3034 | # output, which uses shlex.split to strip options out. |
|
3014 | 3035 | var,_ = args.split('=',1) |
|
3015 | 3036 | var = var.strip() |
|
3016 | 3037 | # But the the command has to be extracted from the original input |
|
3017 | 3038 | # parameter_s, not on what parse_options returns, to avoid the |
|
3018 | 3039 | # quote stripping which shlex.split performs on it. |
|
3019 | 3040 | _,cmd = parameter_s.split('=',1) |
|
3020 | 3041 | except ValueError: |
|
3021 | 3042 | var,cmd = '','' |
|
3022 | 3043 | # If all looks ok, proceed |
|
3023 | 3044 | out,err = self.shell.getoutputerror(cmd) |
|
3024 | 3045 | if err: |
|
3025 | 3046 | print >> Term.cerr,err |
|
3026 | 3047 | if opts.has_key('l'): |
|
3027 | 3048 | out = SList(out.split('\n')) |
|
3028 | 3049 | else: |
|
3029 | 3050 | out = LSString(out) |
|
3030 | 3051 | if opts.has_key('v'): |
|
3031 | 3052 | print '%s ==\n%s' % (var,pformat(out)) |
|
3032 | 3053 | if var: |
|
3033 | 3054 | self.shell.user_ns.update({var:out}) |
|
3034 | 3055 | else: |
|
3035 | 3056 | return out |
|
3036 | 3057 | |
|
3037 | 3058 | def magic_sx(self, parameter_s=''): |
|
3038 | 3059 | """Shell execute - run a shell command and capture its output. |
|
3039 | 3060 | |
|
3040 | 3061 | %sx command |
|
3041 | 3062 | |
|
3042 | 3063 | IPython will run the given command using commands.getoutput(), and |
|
3043 | 3064 | return the result formatted as a list (split on '\\n'). Since the |
|
3044 | 3065 | output is _returned_, it will be stored in ipython's regular output |
|
3045 | 3066 | cache Out[N] and in the '_N' automatic variables. |
|
3046 | 3067 | |
|
3047 | 3068 | Notes: |
|
3048 | 3069 | |
|
3049 | 3070 | 1) If an input line begins with '!!', then %sx is automatically |
|
3050 | 3071 | invoked. That is, while: |
|
3051 | 3072 | !ls |
|
3052 | 3073 | causes ipython to simply issue system('ls'), typing |
|
3053 | 3074 | !!ls |
|
3054 | 3075 | is a shorthand equivalent to: |
|
3055 | 3076 | %sx ls |
|
3056 | 3077 | |
|
3057 | 3078 | 2) %sx differs from %sc in that %sx automatically splits into a list, |
|
3058 | 3079 | like '%sc -l'. The reason for this is to make it as easy as possible |
|
3059 | 3080 | to process line-oriented shell output via further python commands. |
|
3060 | 3081 | %sc is meant to provide much finer control, but requires more |
|
3061 | 3082 | typing. |
|
3062 | 3083 | |
|
3063 | 3084 | 3) Just like %sc -l, this is a list with special attributes: |
|
3064 | 3085 | |
|
3065 | 3086 | .l (or .list) : value as list. |
|
3066 | 3087 | .n (or .nlstr): value as newline-separated string. |
|
3067 | 3088 | .s (or .spstr): value as whitespace-separated string. |
|
3068 | 3089 | |
|
3069 | 3090 | This is very useful when trying to use such lists as arguments to |
|
3070 | 3091 | system commands.""" |
|
3071 | 3092 | |
|
3072 | 3093 | if parameter_s: |
|
3073 | 3094 | out,err = self.shell.getoutputerror(parameter_s) |
|
3074 | 3095 | if err: |
|
3075 | 3096 | print >> Term.cerr,err |
|
3076 | 3097 | return SList(out.split('\n')) |
|
3077 | 3098 | |
|
3078 | 3099 | def magic_bg(self, parameter_s=''): |
|
3079 | 3100 | """Run a job in the background, in a separate thread. |
|
3080 | 3101 | |
|
3081 | 3102 | For example, |
|
3082 | 3103 | |
|
3083 | 3104 | %bg myfunc(x,y,z=1) |
|
3084 | 3105 | |
|
3085 | 3106 | will execute 'myfunc(x,y,z=1)' in a background thread. As soon as the |
|
3086 | 3107 | execution starts, a message will be printed indicating the job |
|
3087 | 3108 | number. If your job number is 5, you can use |
|
3088 | 3109 | |
|
3089 | 3110 | myvar = jobs.result(5) or myvar = jobs[5].result |
|
3090 | 3111 | |
|
3091 | 3112 | to assign this result to variable 'myvar'. |
|
3092 | 3113 | |
|
3093 | 3114 | IPython has a job manager, accessible via the 'jobs' object. You can |
|
3094 | 3115 | type jobs? to get more information about it, and use jobs.<TAB> to see |
|
3095 | 3116 | its attributes. All attributes not starting with an underscore are |
|
3096 | 3117 | meant for public use. |
|
3097 | 3118 | |
|
3098 | 3119 | In particular, look at the jobs.new() method, which is used to create |
|
3099 | 3120 | new jobs. This magic %bg function is just a convenience wrapper |
|
3100 | 3121 | around jobs.new(), for expression-based jobs. If you want to create a |
|
3101 | 3122 | new job with an explicit function object and arguments, you must call |
|
3102 | 3123 | jobs.new() directly. |
|
3103 | 3124 | |
|
3104 | 3125 | The jobs.new docstring also describes in detail several important |
|
3105 | 3126 | caveats associated with a thread-based model for background job |
|
3106 | 3127 | execution. Type jobs.new? for details. |
|
3107 | 3128 | |
|
3108 | 3129 | You can check the status of all jobs with jobs.status(). |
|
3109 | 3130 | |
|
3110 | 3131 | The jobs variable is set by IPython into the Python builtin namespace. |
|
3111 | 3132 | If you ever declare a variable named 'jobs', you will shadow this |
|
3112 | 3133 | name. You can either delete your global jobs variable to regain |
|
3113 | 3134 | access to the job manager, or make a new name and assign it manually |
|
3114 | 3135 | to the manager (stored in IPython's namespace). For example, to |
|
3115 | 3136 | assign the job manager to the Jobs name, use: |
|
3116 | 3137 | |
|
3117 | 3138 | Jobs = __builtins__.jobs""" |
|
3118 | 3139 | |
|
3119 | 3140 | self.shell.jobs.new(parameter_s,self.shell.user_ns) |
|
3120 | 3141 | |
|
3121 | 3142 | def magic_r(self, parameter_s=''): |
|
3122 | 3143 | """Repeat previous input. |
|
3123 | 3144 | |
|
3124 | 3145 | Note: Consider using the more powerfull %rep instead! |
|
3125 | 3146 | |
|
3126 | 3147 | If given an argument, repeats the previous command which starts with |
|
3127 | 3148 | the same string, otherwise it just repeats the previous input. |
|
3128 | 3149 | |
|
3129 | 3150 | Shell escaped commands (with ! as first character) are not recognized |
|
3130 | 3151 | by this system, only pure python code and magic commands. |
|
3131 | 3152 | """ |
|
3132 | 3153 | |
|
3133 | 3154 | start = parameter_s.strip() |
|
3134 | 3155 | esc_magic = self.shell.ESC_MAGIC |
|
3135 | 3156 | # Identify magic commands even if automagic is on (which means |
|
3136 | 3157 | # the in-memory version is different from that typed by the user). |
|
3137 | 3158 | if self.shell.rc.automagic: |
|
3138 | 3159 | start_magic = esc_magic+start |
|
3139 | 3160 | else: |
|
3140 | 3161 | start_magic = start |
|
3141 | 3162 | # Look through the input history in reverse |
|
3142 | 3163 | for n in range(len(self.shell.input_hist)-2,0,-1): |
|
3143 | 3164 | input = self.shell.input_hist[n] |
|
3144 | 3165 | # skip plain 'r' lines so we don't recurse to infinity |
|
3145 | 3166 | if input != '_ip.magic("r")\n' and \ |
|
3146 | 3167 | (input.startswith(start) or input.startswith(start_magic)): |
|
3147 | 3168 | #print 'match',`input` # dbg |
|
3148 | 3169 | print 'Executing:',input, |
|
3149 | 3170 | self.shell.runlines(input) |
|
3150 | 3171 | return |
|
3151 | 3172 | print 'No previous input matching `%s` found.' % start |
|
3152 | 3173 | |
|
3153 | 3174 | |
|
3154 | 3175 | def magic_bookmark(self, parameter_s=''): |
|
3155 | 3176 | """Manage IPython's bookmark system. |
|
3156 | 3177 | |
|
3157 | 3178 | %bookmark <name> - set bookmark to current dir |
|
3158 | 3179 | %bookmark <name> <dir> - set bookmark to <dir> |
|
3159 | 3180 | %bookmark -l - list all bookmarks |
|
3160 | 3181 | %bookmark -d <name> - remove bookmark |
|
3161 | 3182 | %bookmark -r - remove all bookmarks |
|
3162 | 3183 | |
|
3163 | 3184 | You can later on access a bookmarked folder with: |
|
3164 | 3185 | %cd -b <name> |
|
3165 | 3186 | or simply '%cd <name>' if there is no directory called <name> AND |
|
3166 | 3187 | there is such a bookmark defined. |
|
3167 | 3188 | |
|
3168 | 3189 | Your bookmarks persist through IPython sessions, but they are |
|
3169 | 3190 | associated with each profile.""" |
|
3170 | 3191 | |
|
3171 | 3192 | opts,args = self.parse_options(parameter_s,'drl',mode='list') |
|
3172 | 3193 | if len(args) > 2: |
|
3173 | 3194 | raise UsageError("%bookmark: too many arguments") |
|
3174 | 3195 | |
|
3175 | 3196 | bkms = self.db.get('bookmarks',{}) |
|
3176 | 3197 | |
|
3177 | 3198 | if opts.has_key('d'): |
|
3178 | 3199 | try: |
|
3179 | 3200 | todel = args[0] |
|
3180 | 3201 | except IndexError: |
|
3181 | 3202 | raise UsageError( |
|
3182 | 3203 | "%bookmark -d: must provide a bookmark to delete") |
|
3183 | 3204 | else: |
|
3184 | 3205 | try: |
|
3185 | 3206 | del bkms[todel] |
|
3186 | 3207 | except KeyError: |
|
3187 | 3208 | raise UsageError( |
|
3188 | 3209 | "%%bookmark -d: Can't delete bookmark '%s'" % todel) |
|
3189 | 3210 | |
|
3190 | 3211 | elif opts.has_key('r'): |
|
3191 | 3212 | bkms = {} |
|
3192 | 3213 | elif opts.has_key('l'): |
|
3193 | 3214 | bks = bkms.keys() |
|
3194 | 3215 | bks.sort() |
|
3195 | 3216 | if bks: |
|
3196 | 3217 | size = max(map(len,bks)) |
|
3197 | 3218 | else: |
|
3198 | 3219 | size = 0 |
|
3199 | 3220 | fmt = '%-'+str(size)+'s -> %s' |
|
3200 | 3221 | print 'Current bookmarks:' |
|
3201 | 3222 | for bk in bks: |
|
3202 | 3223 | print fmt % (bk,bkms[bk]) |
|
3203 | 3224 | else: |
|
3204 | 3225 | if not args: |
|
3205 | 3226 | raise UsageError("%bookmark: You must specify the bookmark name") |
|
3206 | 3227 | elif len(args)==1: |
|
3207 | 3228 | bkms[args[0]] = os.getcwd() |
|
3208 | 3229 | elif len(args)==2: |
|
3209 | 3230 | bkms[args[0]] = args[1] |
|
3210 | 3231 | self.db['bookmarks'] = bkms |
|
3211 | 3232 | |
|
3212 | 3233 | def magic_pycat(self, parameter_s=''): |
|
3213 | 3234 | """Show a syntax-highlighted file through a pager. |
|
3214 | 3235 | |
|
3215 | 3236 | This magic is similar to the cat utility, but it will assume the file |
|
3216 | 3237 | to be Python source and will show it with syntax highlighting. """ |
|
3217 | 3238 | |
|
3218 | 3239 | try: |
|
3219 | 3240 | filename = get_py_filename(parameter_s) |
|
3220 | 3241 | cont = file_read(filename) |
|
3221 | 3242 | except IOError: |
|
3222 | 3243 | try: |
|
3223 | 3244 | cont = eval(parameter_s,self.user_ns) |
|
3224 | 3245 | except NameError: |
|
3225 | 3246 | cont = None |
|
3226 | 3247 | if cont is None: |
|
3227 | 3248 | print "Error: no such file or variable" |
|
3228 | 3249 | return |
|
3229 | 3250 | |
|
3230 | 3251 | page(self.shell.pycolorize(cont), |
|
3231 | 3252 | screen_lines=self.shell.rc.screen_length) |
|
3232 | 3253 | |
|
3233 | 3254 | def magic_cpaste(self, parameter_s=''): |
|
3234 | 3255 | """Allows you to paste & execute a pre-formatted code block from clipboard. |
|
3235 | 3256 | |
|
3236 | 3257 | You must terminate the block with '--' (two minus-signs) alone on the |
|
3237 | 3258 | line. You can also provide your own sentinel with '%paste -s %%' ('%%' |
|
3238 | 3259 | is the new sentinel for this operation) |
|
3239 | 3260 | |
|
3240 | 3261 | The block is dedented prior to execution to enable execution of method |
|
3241 | 3262 | definitions. '>' and '+' characters at the beginning of a line are |
|
3242 | 3263 | ignored, to allow pasting directly from e-mails, diff files and |
|
3243 | 3264 | doctests (the '...' continuation prompt is also stripped). The |
|
3244 | 3265 | executed block is also assigned to variable named 'pasted_block' for |
|
3245 | 3266 | later editing with '%edit pasted_block'. |
|
3246 | 3267 | |
|
3247 | 3268 | You can also pass a variable name as an argument, e.g. '%cpaste foo'. |
|
3248 | 3269 | This assigns the pasted block to variable 'foo' as string, without |
|
3249 | 3270 | dedenting or executing it (preceding >>> and + is still stripped) |
|
3250 | 3271 | |
|
3251 | 3272 | '%cpaste -r' re-executes the block previously entered by cpaste. |
|
3252 | 3273 | |
|
3253 | 3274 | Do not be alarmed by garbled output on Windows (it's a readline bug). |
|
3254 | 3275 | Just press enter and type -- (and press enter again) and the block |
|
3255 | 3276 | will be what was just pasted. |
|
3256 | 3277 | |
|
3257 | 3278 | IPython statements (magics, shell escapes) are not supported (yet). |
|
3258 | 3279 | """ |
|
3259 | 3280 | opts,args = self.parse_options(parameter_s,'rs:',mode='string') |
|
3260 | 3281 | par = args.strip() |
|
3261 | 3282 | if opts.has_key('r'): |
|
3262 | 3283 | b = self.user_ns.get('pasted_block', None) |
|
3263 | 3284 | if b is None: |
|
3264 | 3285 | raise UsageError('No previous pasted block available') |
|
3265 | 3286 | print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b)) |
|
3266 | 3287 | exec b in self.user_ns |
|
3267 | 3288 | return |
|
3268 | 3289 | |
|
3269 | 3290 | sentinel = opts.get('s','--') |
|
3270 | 3291 | |
|
3271 | 3292 | # Regular expressions that declare text we strip from the input: |
|
3272 | 3293 | strip_re = [r'^\s*In \[\d+\]:', # IPython input prompt |
|
3273 | 3294 | r'^\s*(\s?>)+', # Python input prompt |
|
3274 | 3295 | r'^\s*\.{3,}', # Continuation prompts |
|
3275 | 3296 | r'^\++', |
|
3276 | 3297 | ] |
|
3277 | 3298 | |
|
3278 | 3299 | strip_from_start = map(re.compile,strip_re) |
|
3279 | 3300 | |
|
3280 | 3301 | from IPython import iplib |
|
3281 | 3302 | lines = [] |
|
3282 | 3303 | print "Pasting code; enter '%s' alone on the line to stop." % sentinel |
|
3283 | 3304 | while 1: |
|
3284 | 3305 | l = iplib.raw_input_original(':') |
|
3285 | 3306 | if l ==sentinel: |
|
3286 | 3307 | break |
|
3287 | 3308 | |
|
3288 | 3309 | for pat in strip_from_start: |
|
3289 | 3310 | l = pat.sub('',l) |
|
3290 | 3311 | lines.append(l) |
|
3291 | 3312 | |
|
3292 | 3313 | block = "\n".join(lines) + '\n' |
|
3293 | 3314 | #print "block:\n",block |
|
3294 | 3315 | if not par: |
|
3295 | 3316 | b = textwrap.dedent(block) |
|
3296 | 3317 | self.user_ns['pasted_block'] = b |
|
3297 | 3318 | exec b in self.user_ns |
|
3298 | 3319 | else: |
|
3299 | 3320 | self.user_ns[par] = SList(block.splitlines()) |
|
3300 | 3321 | print "Block assigned to '%s'" % par |
|
3301 | 3322 | |
|
3302 | 3323 | def magic_quickref(self,arg): |
|
3303 | 3324 | """ Show a quick reference sheet """ |
|
3304 | 3325 | import IPython.usage |
|
3305 | 3326 | qr = IPython.usage.quick_reference + self.magic_magic('-brief') |
|
3306 | 3327 | |
|
3307 | 3328 | page(qr) |
|
3308 | 3329 | |
|
3309 | 3330 | def magic_upgrade(self,arg): |
|
3310 | 3331 | """ Upgrade your IPython installation |
|
3311 | 3332 | |
|
3312 | 3333 | This will copy the config files that don't yet exist in your |
|
3313 | 3334 | ipython dir from the system config dir. Use this after upgrading |
|
3314 | 3335 | IPython if you don't wish to delete your .ipython dir. |
|
3315 | 3336 | |
|
3316 | 3337 | Call with -nolegacy to get rid of ipythonrc* files (recommended for |
|
3317 | 3338 | new users) |
|
3318 | 3339 | |
|
3319 | 3340 | """ |
|
3320 | 3341 | ip = self.getapi() |
|
3321 | 3342 | ipinstallation = path(IPython.__file__).dirname() |
|
3322 | 3343 | upgrade_script = '%s "%s"' % (sys.executable,ipinstallation / 'upgrade_dir.py') |
|
3323 | 3344 | src_config = ipinstallation / 'UserConfig' |
|
3324 | 3345 | userdir = path(ip.options.ipythondir) |
|
3325 | 3346 | cmd = '%s "%s" "%s"' % (upgrade_script, src_config, userdir) |
|
3326 | 3347 | print ">",cmd |
|
3327 | 3348 | shell(cmd) |
|
3328 | 3349 | if arg == '-nolegacy': |
|
3329 | 3350 | legacy = userdir.files('ipythonrc*') |
|
3330 | 3351 | print "Nuking legacy files:",legacy |
|
3331 | 3352 | |
|
3332 | 3353 | [p.remove() for p in legacy] |
|
3333 | 3354 | suffix = (sys.platform == 'win32' and '.ini' or '') |
|
3334 | 3355 | (userdir / ('ipythonrc' + suffix)).write_text('# Empty, see ipy_user_conf.py\n') |
|
3335 | 3356 | |
|
3336 | 3357 | |
|
3337 | 3358 | def magic_doctest_mode(self,parameter_s=''): |
|
3338 | 3359 | """Toggle doctest mode on and off. |
|
3339 | 3360 | |
|
3340 | 3361 | This mode allows you to toggle the prompt behavior between normal |
|
3341 | 3362 | IPython prompts and ones that are as similar to the default IPython |
|
3342 | 3363 | interpreter as possible. |
|
3343 | 3364 | |
|
3344 | 3365 | It also supports the pasting of code snippets that have leading '>>>' |
|
3345 | 3366 | and '...' prompts in them. This means that you can paste doctests from |
|
3346 | 3367 | files or docstrings (even if they have leading whitespace), and the |
|
3347 | 3368 | code will execute correctly. You can then use '%history -tn' to see |
|
3348 | 3369 | the translated history without line numbers; this will give you the |
|
3349 | 3370 | input after removal of all the leading prompts and whitespace, which |
|
3350 | 3371 | can be pasted back into an editor. |
|
3351 | 3372 | |
|
3352 | 3373 | With these features, you can switch into this mode easily whenever you |
|
3353 | 3374 | need to do testing and changes to doctests, without having to leave |
|
3354 | 3375 | your existing IPython session. |
|
3355 | 3376 | """ |
|
3356 | 3377 | |
|
3357 | 3378 | # XXX - Fix this to have cleaner activate/deactivate calls. |
|
3358 | 3379 | from IPython.Extensions import InterpreterPasteInput as ipaste |
|
3359 | 3380 | from IPython.ipstruct import Struct |
|
3360 | 3381 | |
|
3361 | 3382 | # Shorthands |
|
3362 | 3383 | shell = self.shell |
|
3363 | 3384 | oc = shell.outputcache |
|
3364 | 3385 | rc = shell.rc |
|
3365 | 3386 | meta = shell.meta |
|
3366 | 3387 | # dstore is a data store kept in the instance metadata bag to track any |
|
3367 | 3388 | # changes we make, so we can undo them later. |
|
3368 | 3389 | dstore = meta.setdefault('doctest_mode',Struct()) |
|
3369 | 3390 | save_dstore = dstore.setdefault |
|
3370 | 3391 | |
|
3371 | 3392 | # save a few values we'll need to recover later |
|
3372 | 3393 | mode = save_dstore('mode',False) |
|
3373 | 3394 | save_dstore('rc_pprint',rc.pprint) |
|
3374 | 3395 | save_dstore('xmode',shell.InteractiveTB.mode) |
|
3375 | 3396 | save_dstore('rc_separate_out',rc.separate_out) |
|
3376 | 3397 | save_dstore('rc_separate_out2',rc.separate_out2) |
|
3377 | 3398 | save_dstore('rc_prompts_pad_left',rc.prompts_pad_left) |
|
3378 | 3399 | save_dstore('rc_separate_in',rc.separate_in) |
|
3379 | 3400 | |
|
3380 | 3401 | if mode == False: |
|
3381 | 3402 | # turn on |
|
3382 | 3403 | ipaste.activate_prefilter() |
|
3383 | 3404 | |
|
3384 | 3405 | oc.prompt1.p_template = '>>> ' |
|
3385 | 3406 | oc.prompt2.p_template = '... ' |
|
3386 | 3407 | oc.prompt_out.p_template = '' |
|
3387 | 3408 | |
|
3388 | 3409 | # Prompt separators like plain python |
|
3389 | 3410 | oc.input_sep = oc.prompt1.sep = '' |
|
3390 | 3411 | oc.output_sep = '' |
|
3391 | 3412 | oc.output_sep2 = '' |
|
3392 | 3413 | |
|
3393 | 3414 | oc.prompt1.pad_left = oc.prompt2.pad_left = \ |
|
3394 | 3415 | oc.prompt_out.pad_left = False |
|
3395 | 3416 | |
|
3396 | 3417 | rc.pprint = False |
|
3397 | 3418 | |
|
3398 | 3419 | shell.magic_xmode('Plain') |
|
3399 | 3420 | |
|
3400 | 3421 | else: |
|
3401 | 3422 | # turn off |
|
3402 | 3423 | ipaste.deactivate_prefilter() |
|
3403 | 3424 | |
|
3404 | 3425 | oc.prompt1.p_template = rc.prompt_in1 |
|
3405 | 3426 | oc.prompt2.p_template = rc.prompt_in2 |
|
3406 | 3427 | oc.prompt_out.p_template = rc.prompt_out |
|
3407 | 3428 | |
|
3408 | 3429 | oc.input_sep = oc.prompt1.sep = dstore.rc_separate_in |
|
3409 | 3430 | |
|
3410 | 3431 | oc.output_sep = dstore.rc_separate_out |
|
3411 | 3432 | oc.output_sep2 = dstore.rc_separate_out2 |
|
3412 | 3433 | |
|
3413 | 3434 | oc.prompt1.pad_left = oc.prompt2.pad_left = \ |
|
3414 | 3435 | oc.prompt_out.pad_left = dstore.rc_prompts_pad_left |
|
3415 | 3436 | |
|
3416 | 3437 | rc.pprint = dstore.rc_pprint |
|
3417 | 3438 | |
|
3418 | 3439 | shell.magic_xmode(dstore.xmode) |
|
3419 | 3440 | |
|
3420 | 3441 | # Store new mode and inform |
|
3421 | 3442 | dstore.mode = bool(1-int(mode)) |
|
3422 | 3443 | print 'Doctest mode is:', |
|
3423 | 3444 | print ['OFF','ON'][dstore.mode] |
|
3424 | 3445 | |
|
3425 | 3446 | # end Magic |
@@ -1,622 +1,622 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """ |
|
3 | 3 | Classes for handling input/output prompts. |
|
4 | 4 | """ |
|
5 | 5 | |
|
6 | 6 | #***************************************************************************** |
|
7 | 7 | # Copyright (C) 2008-2009 The IPython Development Team |
|
8 | 8 | # Copyright (C) 2001-2007 Fernando Perez <fperez@colorado.edu> |
|
9 | 9 | # |
|
10 | 10 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | 11 | # the file COPYING, distributed as part of this software. |
|
12 | 12 | #***************************************************************************** |
|
13 | 13 | |
|
14 | 14 | #**************************************************************************** |
|
15 | 15 | # Required modules |
|
16 | 16 | import __builtin__ |
|
17 | 17 | import os |
|
18 | 18 | import socket |
|
19 | 19 | import sys |
|
20 | 20 | import time |
|
21 | 21 | |
|
22 | 22 | # IPython's own |
|
23 | 23 | from IPython import ColorANSI |
|
24 | 24 | from IPython import Release |
|
25 | 25 | from IPython.external.Itpl import ItplNS |
|
26 | 26 | from IPython.ipapi import TryNext |
|
27 | 27 | from IPython.ipstruct import Struct |
|
28 | 28 | from IPython.macro import Macro |
|
29 | 29 | |
|
30 | 30 | from IPython.genutils import * |
|
31 | 31 | |
|
32 | 32 | #**************************************************************************** |
|
33 | 33 | #Color schemes for Prompts. |
|
34 | 34 | |
|
35 | 35 | PromptColors = ColorANSI.ColorSchemeTable() |
|
36 | 36 | InputColors = ColorANSI.InputTermColors # just a shorthand |
|
37 | 37 | Colors = ColorANSI.TermColors # just a shorthand |
|
38 | 38 | |
|
39 | 39 | PromptColors.add_scheme(ColorANSI.ColorScheme( |
|
40 | 40 | 'NoColor', |
|
41 | 41 | in_prompt = InputColors.NoColor, # Input prompt |
|
42 | 42 | in_number = InputColors.NoColor, # Input prompt number |
|
43 | 43 | in_prompt2 = InputColors.NoColor, # Continuation prompt |
|
44 | 44 | in_normal = InputColors.NoColor, # color off (usu. Colors.Normal) |
|
45 | 45 | |
|
46 | 46 | out_prompt = Colors.NoColor, # Output prompt |
|
47 | 47 | out_number = Colors.NoColor, # Output prompt number |
|
48 | 48 | |
|
49 | 49 | normal = Colors.NoColor # color off (usu. Colors.Normal) |
|
50 | 50 | )) |
|
51 | 51 | |
|
52 | 52 | # make some schemes as instances so we can copy them for modification easily: |
|
53 | 53 | __PColLinux = ColorANSI.ColorScheme( |
|
54 | 54 | 'Linux', |
|
55 | 55 | in_prompt = InputColors.Green, |
|
56 | 56 | in_number = InputColors.LightGreen, |
|
57 | 57 | in_prompt2 = InputColors.Green, |
|
58 | 58 | in_normal = InputColors.Normal, # color off (usu. Colors.Normal) |
|
59 | 59 | |
|
60 | 60 | out_prompt = Colors.Red, |
|
61 | 61 | out_number = Colors.LightRed, |
|
62 | 62 | |
|
63 | 63 | normal = Colors.Normal |
|
64 | 64 | ) |
|
65 | 65 | # Don't forget to enter it into the table! |
|
66 | 66 | PromptColors.add_scheme(__PColLinux) |
|
67 | 67 | |
|
68 | 68 | # Slightly modified Linux for light backgrounds |
|
69 | 69 | __PColLightBG = __PColLinux.copy('LightBG') |
|
70 | 70 | |
|
71 | 71 | __PColLightBG.colors.update( |
|
72 | 72 | in_prompt = InputColors.Blue, |
|
73 | 73 | in_number = InputColors.LightBlue, |
|
74 | 74 | in_prompt2 = InputColors.Blue |
|
75 | 75 | ) |
|
76 | 76 | PromptColors.add_scheme(__PColLightBG) |
|
77 | 77 | |
|
78 | 78 | del Colors,InputColors |
|
79 | 79 | |
|
80 | 80 | #----------------------------------------------------------------------------- |
|
81 | 81 | def multiple_replace(dict, text): |
|
82 | 82 | """ Replace in 'text' all occurences of any key in the given |
|
83 | 83 | dictionary by its corresponding value. Returns the new string.""" |
|
84 | 84 | |
|
85 | 85 | # Function by Xavier Defrang, originally found at: |
|
86 | 86 | # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81330 |
|
87 | 87 | |
|
88 | 88 | # Create a regular expression from the dictionary keys |
|
89 | 89 | regex = re.compile("(%s)" % "|".join(map(re.escape, dict.keys()))) |
|
90 | 90 | # For each match, look-up corresponding value in dictionary |
|
91 | 91 | return regex.sub(lambda mo: dict[mo.string[mo.start():mo.end()]], text) |
|
92 | 92 | |
|
93 | 93 | #----------------------------------------------------------------------------- |
|
94 | 94 | # Special characters that can be used in prompt templates, mainly bash-like |
|
95 | 95 | |
|
96 | 96 | # If $HOME isn't defined (Windows), make it an absurd string so that it can |
|
97 | 97 | # never be expanded out into '~'. Basically anything which can never be a |
|
98 | 98 | # reasonable directory name will do, we just want the $HOME -> '~' operation |
|
99 | 99 | # to become a no-op. We pre-compute $HOME here so it's not done on every |
|
100 | 100 | # prompt call. |
|
101 | 101 | |
|
102 | 102 | # FIXME: |
|
103 | 103 | |
|
104 | 104 | # - This should be turned into a class which does proper namespace management, |
|
105 | 105 | # since the prompt specials need to be evaluated in a certain namespace. |
|
106 | 106 | # Currently it's just globals, which need to be managed manually by code |
|
107 | 107 | # below. |
|
108 | 108 | |
|
109 | 109 | # - I also need to split up the color schemes from the prompt specials |
|
110 | 110 | # somehow. I don't have a clean design for that quite yet. |
|
111 | 111 | |
|
112 | 112 | HOME = os.environ.get("HOME","//////:::::ZZZZZ,,,~~~") |
|
113 | 113 | |
|
114 | 114 | # We precompute a few more strings here for the prompt_specials, which are |
|
115 | 115 | # fixed once ipython starts. This reduces the runtime overhead of computing |
|
116 | 116 | # prompt strings. |
|
117 | 117 | USER = os.environ.get("USER") |
|
118 | 118 | HOSTNAME = socket.gethostname() |
|
119 | 119 | HOSTNAME_SHORT = HOSTNAME.split(".")[0] |
|
120 | 120 | ROOT_SYMBOL = "$#"[os.name=='nt' or os.getuid()==0] |
|
121 | 121 | |
|
122 | 122 | prompt_specials_color = { |
|
123 | 123 | # Prompt/history count |
|
124 | 124 | '%n' : '${self.col_num}' '${self.cache.prompt_count}' '${self.col_p}', |
|
125 | 125 | r'\#': '${self.col_num}' '${self.cache.prompt_count}' '${self.col_p}', |
|
126 | 126 | # Just the prompt counter number, WITHOUT any coloring wrappers, so users |
|
127 | 127 | # can get numbers displayed in whatever color they want. |
|
128 | 128 | r'\N': '${self.cache.prompt_count}', |
|
129 | 129 | # Prompt/history count, with the actual digits replaced by dots. Used |
|
130 | 130 | # mainly in continuation prompts (prompt_in2) |
|
131 | r'\D': '${"."*len(str(self.cache.prompt_count))}', | |
|
131 | r'\D': '${"."*__builtins__.len(__builtins__.str(self.cache.prompt_count))}', | |
|
132 | 132 | # Current working directory |
|
133 | 133 | r'\w': '${os.getcwd()}', |
|
134 | 134 | # Current time |
|
135 | 135 | r'\t' : '${time.strftime("%H:%M:%S")}', |
|
136 | 136 | # Basename of current working directory. |
|
137 | 137 | # (use os.sep to make this portable across OSes) |
|
138 | 138 | r'\W' : '${os.getcwd().split("%s")[-1]}' % os.sep, |
|
139 | 139 | # These X<N> are an extension to the normal bash prompts. They return |
|
140 | 140 | # N terms of the path, after replacing $HOME with '~' |
|
141 | 141 | r'\X0': '${os.getcwd().replace("%s","~")}' % HOME, |
|
142 | 142 | r'\X1': '${self.cwd_filt(1)}', |
|
143 | 143 | r'\X2': '${self.cwd_filt(2)}', |
|
144 | 144 | r'\X3': '${self.cwd_filt(3)}', |
|
145 | 145 | r'\X4': '${self.cwd_filt(4)}', |
|
146 | 146 | r'\X5': '${self.cwd_filt(5)}', |
|
147 | 147 | # Y<N> are similar to X<N>, but they show '~' if it's the directory |
|
148 | 148 | # N+1 in the list. Somewhat like %cN in tcsh. |
|
149 | 149 | r'\Y0': '${self.cwd_filt2(0)}', |
|
150 | 150 | r'\Y1': '${self.cwd_filt2(1)}', |
|
151 | 151 | r'\Y2': '${self.cwd_filt2(2)}', |
|
152 | 152 | r'\Y3': '${self.cwd_filt2(3)}', |
|
153 | 153 | r'\Y4': '${self.cwd_filt2(4)}', |
|
154 | 154 | r'\Y5': '${self.cwd_filt2(5)}', |
|
155 | 155 | # Hostname up to first . |
|
156 | 156 | r'\h': HOSTNAME_SHORT, |
|
157 | 157 | # Full hostname |
|
158 | 158 | r'\H': HOSTNAME, |
|
159 | 159 | # Username of current user |
|
160 | 160 | r'\u': USER, |
|
161 | 161 | # Escaped '\' |
|
162 | 162 | '\\\\': '\\', |
|
163 | 163 | # Newline |
|
164 | 164 | r'\n': '\n', |
|
165 | 165 | # Carriage return |
|
166 | 166 | r'\r': '\r', |
|
167 | 167 | # Release version |
|
168 | 168 | r'\v': Release.version, |
|
169 | 169 | # Root symbol ($ or #) |
|
170 | 170 | r'\$': ROOT_SYMBOL, |
|
171 | 171 | } |
|
172 | 172 | |
|
173 | 173 | # A copy of the prompt_specials dictionary but with all color escapes removed, |
|
174 | 174 | # so we can correctly compute the prompt length for the auto_rewrite method. |
|
175 | 175 | prompt_specials_nocolor = prompt_specials_color.copy() |
|
176 | 176 | prompt_specials_nocolor['%n'] = '${self.cache.prompt_count}' |
|
177 | 177 | prompt_specials_nocolor[r'\#'] = '${self.cache.prompt_count}' |
|
178 | 178 | |
|
179 | 179 | # Add in all the InputTermColors color escapes as valid prompt characters. |
|
180 | 180 | # They all get added as \\C_COLORNAME, so that we don't have any conflicts |
|
181 | 181 | # with a color name which may begin with a letter used by any other of the |
|
182 | 182 | # allowed specials. This of course means that \\C will never be allowed for |
|
183 | 183 | # anything else. |
|
184 | 184 | input_colors = ColorANSI.InputTermColors |
|
185 | 185 | for _color in dir(input_colors): |
|
186 | 186 | if _color[0] != '_': |
|
187 | 187 | c_name = r'\C_'+_color |
|
188 | 188 | prompt_specials_color[c_name] = getattr(input_colors,_color) |
|
189 | 189 | prompt_specials_nocolor[c_name] = '' |
|
190 | 190 | |
|
191 | 191 | # we default to no color for safety. Note that prompt_specials is a global |
|
192 | 192 | # variable used by all prompt objects. |
|
193 | 193 | prompt_specials = prompt_specials_nocolor |
|
194 | 194 | |
|
195 | 195 | #----------------------------------------------------------------------------- |
|
196 | 196 | def str_safe(arg): |
|
197 | 197 | """Convert to a string, without ever raising an exception. |
|
198 | 198 | |
|
199 | 199 | If str(arg) fails, <ERROR: ... > is returned, where ... is the exception |
|
200 | 200 | error message.""" |
|
201 | 201 | |
|
202 | 202 | try: |
|
203 | 203 | out = str(arg) |
|
204 | 204 | except UnicodeError: |
|
205 | 205 | try: |
|
206 | 206 | out = arg.encode('utf_8','replace') |
|
207 | 207 | except Exception,msg: |
|
208 | 208 | # let's keep this little duplication here, so that the most common |
|
209 | 209 | # case doesn't suffer from a double try wrapping. |
|
210 | 210 | out = '<ERROR: %s>' % msg |
|
211 | 211 | except Exception,msg: |
|
212 | 212 | out = '<ERROR: %s>' % msg |
|
213 | 213 | return out |
|
214 | 214 | |
|
215 | 215 | class BasePrompt(object): |
|
216 | 216 | """Interactive prompt similar to Mathematica's.""" |
|
217 | 217 | |
|
218 | 218 | def _get_p_template(self): |
|
219 | 219 | return self._p_template |
|
220 | 220 | |
|
221 | 221 | def _set_p_template(self,val): |
|
222 | 222 | self._p_template = val |
|
223 | 223 | self.set_p_str() |
|
224 | 224 | |
|
225 | 225 | p_template = property(_get_p_template,_set_p_template, |
|
226 | 226 | doc='Template for prompt string creation') |
|
227 | 227 | |
|
228 | 228 | def __init__(self,cache,sep,prompt,pad_left=False): |
|
229 | 229 | |
|
230 | 230 | # Hack: we access information about the primary prompt through the |
|
231 | 231 | # cache argument. We need this, because we want the secondary prompt |
|
232 | 232 | # to be aligned with the primary one. Color table info is also shared |
|
233 | 233 | # by all prompt classes through the cache. Nice OO spaghetti code! |
|
234 | 234 | self.cache = cache |
|
235 | 235 | self.sep = sep |
|
236 | 236 | |
|
237 | 237 | # regexp to count the number of spaces at the end of a prompt |
|
238 | 238 | # expression, useful for prompt auto-rewriting |
|
239 | 239 | self.rspace = re.compile(r'(\s*)$') |
|
240 | 240 | # Flag to left-pad prompt strings to match the length of the primary |
|
241 | 241 | # prompt |
|
242 | 242 | self.pad_left = pad_left |
|
243 | 243 | |
|
244 | 244 | # Set template to create each actual prompt (where numbers change). |
|
245 | 245 | # Use a property |
|
246 | 246 | self.p_template = prompt |
|
247 | 247 | self.set_p_str() |
|
248 | 248 | |
|
249 | 249 | def set_p_str(self): |
|
250 | 250 | """ Set the interpolating prompt strings. |
|
251 | 251 | |
|
252 | 252 | This must be called every time the color settings change, because the |
|
253 | 253 | prompt_specials global may have changed.""" |
|
254 | 254 | |
|
255 | 255 | import os,time # needed in locals for prompt string handling |
|
256 | 256 | loc = locals() |
|
257 | 257 | try: |
|
258 | 258 | self.p_str = ItplNS('%s%s%s' % |
|
259 | 259 | ('${self.sep}${self.col_p}', |
|
260 | 260 | multiple_replace(prompt_specials, self.p_template), |
|
261 | 261 | '${self.col_norm}'),self.cache.user_ns,loc) |
|
262 | 262 | |
|
263 | 263 | self.p_str_nocolor = ItplNS(multiple_replace(prompt_specials_nocolor, |
|
264 | 264 | self.p_template), |
|
265 | 265 | self.cache.user_ns,loc) |
|
266 | 266 | except: |
|
267 | 267 | print "Illegal prompt template (check $ usage!):",self.p_template |
|
268 | 268 | self.p_str = self.p_template |
|
269 | 269 | self.p_str_nocolor = self.p_template |
|
270 | 270 | |
|
271 | 271 | def write(self,msg): # dbg |
|
272 | 272 | sys.stdout.write(msg) |
|
273 | 273 | return '' |
|
274 | 274 | |
|
275 | 275 | def __str__(self): |
|
276 | 276 | """Return a string form of the prompt. |
|
277 | 277 | |
|
278 | 278 | This for is useful for continuation and output prompts, since it is |
|
279 | 279 | left-padded to match lengths with the primary one (if the |
|
280 | 280 | self.pad_left attribute is set).""" |
|
281 | 281 | |
|
282 | 282 | out_str = str_safe(self.p_str) |
|
283 | 283 | if self.pad_left: |
|
284 | 284 | # We must find the amount of padding required to match lengths, |
|
285 | 285 | # taking the color escapes (which are invisible on-screen) into |
|
286 | 286 | # account. |
|
287 | 287 | esc_pad = len(out_str) - len(str_safe(self.p_str_nocolor)) |
|
288 | 288 | format = '%%%ss' % (len(str(self.cache.last_prompt))+esc_pad) |
|
289 | 289 | return format % out_str |
|
290 | 290 | else: |
|
291 | 291 | return out_str |
|
292 | 292 | |
|
293 | 293 | # these path filters are put in as methods so that we can control the |
|
294 | 294 | # namespace where the prompt strings get evaluated |
|
295 | 295 | def cwd_filt(self,depth): |
|
296 | 296 | """Return the last depth elements of the current working directory. |
|
297 | 297 | |
|
298 | 298 | $HOME is always replaced with '~'. |
|
299 | 299 | If depth==0, the full path is returned.""" |
|
300 | 300 | |
|
301 | 301 | cwd = os.getcwd().replace(HOME,"~") |
|
302 | 302 | out = os.sep.join(cwd.split(os.sep)[-depth:]) |
|
303 | 303 | if out: |
|
304 | 304 | return out |
|
305 | 305 | else: |
|
306 | 306 | return os.sep |
|
307 | 307 | |
|
308 | 308 | def cwd_filt2(self,depth): |
|
309 | 309 | """Return the last depth elements of the current working directory. |
|
310 | 310 | |
|
311 | 311 | $HOME is always replaced with '~'. |
|
312 | 312 | If depth==0, the full path is returned.""" |
|
313 | 313 | |
|
314 | 314 | full_cwd = os.getcwd() |
|
315 | 315 | cwd = full_cwd.replace(HOME,"~").split(os.sep) |
|
316 | 316 | if '~' in cwd and len(cwd) == depth+1: |
|
317 | 317 | depth += 1 |
|
318 | 318 | drivepart = '' |
|
319 | 319 | if sys.platform == 'win32' and len(cwd) > depth: |
|
320 | 320 | drivepart = os.path.splitdrive(full_cwd)[0] |
|
321 | 321 | out = drivepart + '/'.join(cwd[-depth:]) |
|
322 | 322 | |
|
323 | 323 | if out: |
|
324 | 324 | return out |
|
325 | 325 | else: |
|
326 | 326 | return os.sep |
|
327 | 327 | |
|
328 | 328 | def __nonzero__(self): |
|
329 | 329 | """Implement boolean behavior. |
|
330 | 330 | |
|
331 | 331 | Checks whether the p_str attribute is non-empty""" |
|
332 | 332 | |
|
333 | 333 | return bool(self.p_template) |
|
334 | 334 | |
|
335 | 335 | class Prompt1(BasePrompt): |
|
336 | 336 | """Input interactive prompt similar to Mathematica's.""" |
|
337 | 337 | |
|
338 | 338 | def __init__(self,cache,sep='\n',prompt='In [\\#]: ',pad_left=True): |
|
339 | 339 | BasePrompt.__init__(self,cache,sep,prompt,pad_left) |
|
340 | 340 | |
|
341 | 341 | def set_colors(self): |
|
342 | 342 | self.set_p_str() |
|
343 | 343 | Colors = self.cache.color_table.active_colors # shorthand |
|
344 | 344 | self.col_p = Colors.in_prompt |
|
345 | 345 | self.col_num = Colors.in_number |
|
346 | 346 | self.col_norm = Colors.in_normal |
|
347 | 347 | # We need a non-input version of these escapes for the '--->' |
|
348 | 348 | # auto-call prompts used in the auto_rewrite() method. |
|
349 | 349 | self.col_p_ni = self.col_p.replace('\001','').replace('\002','') |
|
350 | 350 | self.col_norm_ni = Colors.normal |
|
351 | 351 | |
|
352 | 352 | def __str__(self): |
|
353 | 353 | self.cache.prompt_count += 1 |
|
354 | 354 | self.cache.last_prompt = str_safe(self.p_str_nocolor).split('\n')[-1] |
|
355 | 355 | return str_safe(self.p_str) |
|
356 | 356 | |
|
357 | 357 | def auto_rewrite(self): |
|
358 | 358 | """Print a string of the form '--->' which lines up with the previous |
|
359 | 359 | input string. Useful for systems which re-write the user input when |
|
360 | 360 | handling automatically special syntaxes.""" |
|
361 | 361 | |
|
362 | 362 | curr = str(self.cache.last_prompt) |
|
363 | 363 | nrspaces = len(self.rspace.search(curr).group()) |
|
364 | 364 | return '%s%s>%s%s' % (self.col_p_ni,'-'*(len(curr)-nrspaces-1), |
|
365 | 365 | ' '*nrspaces,self.col_norm_ni) |
|
366 | 366 | |
|
367 | 367 | class PromptOut(BasePrompt): |
|
368 | 368 | """Output interactive prompt similar to Mathematica's.""" |
|
369 | 369 | |
|
370 | 370 | def __init__(self,cache,sep='',prompt='Out[\\#]: ',pad_left=True): |
|
371 | 371 | BasePrompt.__init__(self,cache,sep,prompt,pad_left) |
|
372 | 372 | if not self.p_template: |
|
373 | 373 | self.__str__ = lambda: '' |
|
374 | 374 | |
|
375 | 375 | def set_colors(self): |
|
376 | 376 | self.set_p_str() |
|
377 | 377 | Colors = self.cache.color_table.active_colors # shorthand |
|
378 | 378 | self.col_p = Colors.out_prompt |
|
379 | 379 | self.col_num = Colors.out_number |
|
380 | 380 | self.col_norm = Colors.normal |
|
381 | 381 | |
|
382 | 382 | class Prompt2(BasePrompt): |
|
383 | 383 | """Interactive continuation prompt.""" |
|
384 | 384 | |
|
385 | 385 | def __init__(self,cache,prompt=' .\\D.: ',pad_left=True): |
|
386 | 386 | self.cache = cache |
|
387 | 387 | self.p_template = prompt |
|
388 | 388 | self.pad_left = pad_left |
|
389 | 389 | self.set_p_str() |
|
390 | 390 | |
|
391 | 391 | def set_p_str(self): |
|
392 | 392 | import os,time # needed in locals for prompt string handling |
|
393 | 393 | loc = locals() |
|
394 | 394 | self.p_str = ItplNS('%s%s%s' % |
|
395 | 395 | ('${self.col_p2}', |
|
396 | 396 | multiple_replace(prompt_specials, self.p_template), |
|
397 | 397 | '$self.col_norm'), |
|
398 | 398 | self.cache.user_ns,loc) |
|
399 | 399 | self.p_str_nocolor = ItplNS(multiple_replace(prompt_specials_nocolor, |
|
400 | 400 | self.p_template), |
|
401 | 401 | self.cache.user_ns,loc) |
|
402 | 402 | |
|
403 | 403 | def set_colors(self): |
|
404 | 404 | self.set_p_str() |
|
405 | 405 | Colors = self.cache.color_table.active_colors |
|
406 | 406 | self.col_p2 = Colors.in_prompt2 |
|
407 | 407 | self.col_norm = Colors.in_normal |
|
408 | 408 | # FIXME (2004-06-16) HACK: prevent crashes for users who haven't |
|
409 | 409 | # updated their prompt_in2 definitions. Remove eventually. |
|
410 | 410 | self.col_p = Colors.out_prompt |
|
411 | 411 | self.col_num = Colors.out_number |
|
412 | 412 | |
|
413 | 413 | |
|
414 | 414 | #----------------------------------------------------------------------------- |
|
415 | 415 | class CachedOutput: |
|
416 | 416 | """Class for printing output from calculations while keeping a cache of |
|
417 | 417 | reults. It dynamically creates global variables prefixed with _ which |
|
418 | 418 | contain these results. |
|
419 | 419 | |
|
420 | 420 | Meant to be used as a sys.displayhook replacement, providing numbered |
|
421 | 421 | prompts and cache services. |
|
422 | 422 | |
|
423 | 423 | Initialize with initial and final values for cache counter (this defines |
|
424 | 424 | the maximum size of the cache.""" |
|
425 | 425 | |
|
426 | 426 | def __init__(self,shell,cache_size,Pprint, |
|
427 | 427 | colors='NoColor',input_sep='\n', |
|
428 | 428 | output_sep='\n',output_sep2='', |
|
429 | 429 | ps1 = None, ps2 = None,ps_out = None,pad_left=True): |
|
430 | 430 | |
|
431 | 431 | cache_size_min = 3 |
|
432 | 432 | if cache_size <= 0: |
|
433 | 433 | self.do_full_cache = 0 |
|
434 | 434 | cache_size = 0 |
|
435 | 435 | elif cache_size < cache_size_min: |
|
436 | 436 | self.do_full_cache = 0 |
|
437 | 437 | cache_size = 0 |
|
438 | 438 | warn('caching was disabled (min value for cache size is %s).' % |
|
439 | 439 | cache_size_min,level=3) |
|
440 | 440 | else: |
|
441 | 441 | self.do_full_cache = 1 |
|
442 | 442 | |
|
443 | 443 | self.cache_size = cache_size |
|
444 | 444 | self.input_sep = input_sep |
|
445 | 445 | |
|
446 | 446 | # we need a reference to the user-level namespace |
|
447 | 447 | self.shell = shell |
|
448 | 448 | self.user_ns = shell.user_ns |
|
449 | 449 | # and to the user's input |
|
450 | 450 | self.input_hist = shell.input_hist |
|
451 | 451 | # and to the user's logger, for logging output |
|
452 | 452 | self.logger = shell.logger |
|
453 | 453 | |
|
454 | 454 | # Set input prompt strings and colors |
|
455 | 455 | if cache_size == 0: |
|
456 | 456 | if ps1.find('%n') > -1 or ps1.find(r'\#') > -1 \ |
|
457 | 457 | or ps1.find(r'\N') > -1: |
|
458 | 458 | ps1 = '>>> ' |
|
459 | 459 | if ps2.find('%n') > -1 or ps2.find(r'\#') > -1 \ |
|
460 | 460 | or ps2.find(r'\N') > -1: |
|
461 | 461 | ps2 = '... ' |
|
462 | 462 | self.ps1_str = self._set_prompt_str(ps1,'In [\\#]: ','>>> ') |
|
463 | 463 | self.ps2_str = self._set_prompt_str(ps2,' .\\D.: ','... ') |
|
464 | 464 | self.ps_out_str = self._set_prompt_str(ps_out,'Out[\\#]: ','') |
|
465 | 465 | |
|
466 | 466 | self.color_table = PromptColors |
|
467 | 467 | self.prompt1 = Prompt1(self,sep=input_sep,prompt=self.ps1_str, |
|
468 | 468 | pad_left=pad_left) |
|
469 | 469 | self.prompt2 = Prompt2(self,prompt=self.ps2_str,pad_left=pad_left) |
|
470 | 470 | self.prompt_out = PromptOut(self,sep='',prompt=self.ps_out_str, |
|
471 | 471 | pad_left=pad_left) |
|
472 | 472 | self.set_colors(colors) |
|
473 | 473 | |
|
474 | 474 | # other more normal stuff |
|
475 | 475 | # b/c each call to the In[] prompt raises it by 1, even the first. |
|
476 | 476 | self.prompt_count = 0 |
|
477 | 477 | # Store the last prompt string each time, we need it for aligning |
|
478 | 478 | # continuation and auto-rewrite prompts |
|
479 | 479 | self.last_prompt = '' |
|
480 | 480 | self.Pprint = Pprint |
|
481 | 481 | self.output_sep = output_sep |
|
482 | 482 | self.output_sep2 = output_sep2 |
|
483 | 483 | self._,self.__,self.___ = '','','' |
|
484 | 484 | self.pprint_types = map(type,[(),[],{}]) |
|
485 | 485 | |
|
486 | 486 | # these are deliberately global: |
|
487 | 487 | to_user_ns = {'_':self._,'__':self.__,'___':self.___} |
|
488 | 488 | self.user_ns.update(to_user_ns) |
|
489 | 489 | |
|
490 | 490 | def _set_prompt_str(self,p_str,cache_def,no_cache_def): |
|
491 | 491 | if p_str is None: |
|
492 | 492 | if self.do_full_cache: |
|
493 | 493 | return cache_def |
|
494 | 494 | else: |
|
495 | 495 | return no_cache_def |
|
496 | 496 | else: |
|
497 | 497 | return p_str |
|
498 | 498 | |
|
499 | 499 | def set_colors(self,colors): |
|
500 | 500 | """Set the active color scheme and configure colors for the three |
|
501 | 501 | prompt subsystems.""" |
|
502 | 502 | |
|
503 | 503 | # FIXME: the prompt_specials global should be gobbled inside this |
|
504 | 504 | # class instead. Do it when cleaning up the whole 3-prompt system. |
|
505 | 505 | global prompt_specials |
|
506 | 506 | if colors.lower()=='nocolor': |
|
507 | 507 | prompt_specials = prompt_specials_nocolor |
|
508 | 508 | else: |
|
509 | 509 | prompt_specials = prompt_specials_color |
|
510 | 510 | |
|
511 | 511 | self.color_table.set_active_scheme(colors) |
|
512 | 512 | self.prompt1.set_colors() |
|
513 | 513 | self.prompt2.set_colors() |
|
514 | 514 | self.prompt_out.set_colors() |
|
515 | 515 | |
|
516 | 516 | def __call__(self,arg=None): |
|
517 | 517 | """Printing with history cache management. |
|
518 | 518 | |
|
519 | 519 | This is invoked everytime the interpreter needs to print, and is |
|
520 | 520 | activated by setting the variable sys.displayhook to it.""" |
|
521 | 521 | |
|
522 | 522 | # If something injected a '_' variable in __builtin__, delete |
|
523 | 523 | # ipython's automatic one so we don't clobber that. gettext() in |
|
524 | 524 | # particular uses _, so we need to stay away from it. |
|
525 | 525 | if '_' in __builtin__.__dict__: |
|
526 | 526 | try: |
|
527 | 527 | del self.user_ns['_'] |
|
528 | 528 | except KeyError: |
|
529 | 529 | pass |
|
530 | 530 | if arg is not None: |
|
531 | 531 | cout_write = Term.cout.write # fast lookup |
|
532 | 532 | # first handle the cache and counters |
|
533 | 533 | |
|
534 | 534 | # do not print output if input ends in ';' |
|
535 | 535 | try: |
|
536 | 536 | if self.input_hist[self.prompt_count].endswith(';\n'): |
|
537 | 537 | return |
|
538 | 538 | except IndexError: |
|
539 | 539 | # some uses of ipshellembed may fail here |
|
540 | 540 | pass |
|
541 | 541 | # don't use print, puts an extra space |
|
542 | 542 | cout_write(self.output_sep) |
|
543 | 543 | outprompt = self.shell.hooks.generate_output_prompt() |
|
544 | 544 | if self.do_full_cache: |
|
545 | 545 | cout_write(outprompt) |
|
546 | 546 | |
|
547 | 547 | # and now call a possibly user-defined print mechanism |
|
548 | 548 | manipulated_val = self.display(arg) |
|
549 | 549 | |
|
550 | 550 | # user display hooks can change the variable to be stored in |
|
551 | 551 | # output history |
|
552 | 552 | |
|
553 | 553 | if manipulated_val is not None: |
|
554 | 554 | arg = manipulated_val |
|
555 | 555 | |
|
556 | 556 | # avoid recursive reference when displaying _oh/Out |
|
557 | 557 | if arg is not self.user_ns['_oh']: |
|
558 | 558 | self.update(arg) |
|
559 | 559 | |
|
560 | 560 | if self.logger.log_output: |
|
561 | 561 | self.logger.log_write(repr(arg),'output') |
|
562 | 562 | cout_write(self.output_sep2) |
|
563 | 563 | Term.cout.flush() |
|
564 | 564 | |
|
565 | 565 | def _display(self,arg): |
|
566 | 566 | """Default printer method, uses pprint. |
|
567 | 567 | |
|
568 | 568 | Do ip.set_hook("result_display", my_displayhook) for custom result |
|
569 | 569 | display, e.g. when your own objects need special formatting. |
|
570 | 570 | """ |
|
571 | 571 | try: |
|
572 | 572 | return IPython.generics.result_display(arg) |
|
573 | 573 | except TryNext: |
|
574 | 574 | return self.shell.hooks.result_display(arg) |
|
575 | 575 | |
|
576 | 576 | # Assign the default display method: |
|
577 | 577 | display = _display |
|
578 | 578 | |
|
579 | 579 | def update(self,arg): |
|
580 | 580 | #print '***cache_count', self.cache_count # dbg |
|
581 | 581 | if len(self.user_ns['_oh']) >= self.cache_size and self.do_full_cache: |
|
582 | 582 | warn('Output cache limit (currently '+ |
|
583 | 583 | `self.cache_size`+' entries) hit.\n' |
|
584 | 584 | 'Flushing cache and resetting history counter...\n' |
|
585 | 585 | 'The only history variables available will be _,__,___ and _1\n' |
|
586 | 586 | 'with the current result.') |
|
587 | 587 | |
|
588 | 588 | self.flush() |
|
589 | 589 | # Don't overwrite '_' and friends if '_' is in __builtin__ (otherwise |
|
590 | 590 | # we cause buggy behavior for things like gettext). |
|
591 | 591 | if '_' not in __builtin__.__dict__: |
|
592 | 592 | self.___ = self.__ |
|
593 | 593 | self.__ = self._ |
|
594 | 594 | self._ = arg |
|
595 | 595 | self.user_ns.update({'_':self._,'__':self.__,'___':self.___}) |
|
596 | 596 | |
|
597 | 597 | # hackish access to top-level namespace to create _1,_2... dynamically |
|
598 | 598 | to_main = {} |
|
599 | 599 | if self.do_full_cache: |
|
600 | 600 | new_result = '_'+`self.prompt_count` |
|
601 | 601 | to_main[new_result] = arg |
|
602 | 602 | self.user_ns.update(to_main) |
|
603 | 603 | self.user_ns['_oh'][self.prompt_count] = arg |
|
604 | 604 | |
|
605 | 605 | def flush(self): |
|
606 | 606 | if not self.do_full_cache: |
|
607 | 607 | raise ValueError,"You shouldn't have reached the cache flush "\ |
|
608 | 608 | "if full caching is not enabled!" |
|
609 | 609 | # delete auto-generated vars from global namespace |
|
610 | 610 | |
|
611 | 611 | for n in range(1,self.prompt_count + 1): |
|
612 | 612 | key = '_'+`n` |
|
613 | 613 | try: |
|
614 | 614 | del self.user_ns[key] |
|
615 | 615 | except: pass |
|
616 | 616 | self.user_ns['_oh'].clear() |
|
617 | 617 | |
|
618 | 618 | if '_' not in __builtin__.__dict__: |
|
619 | 619 | self.user_ns.update({'_':None,'__':None, '___':None}) |
|
620 | 620 | import gc |
|
621 | 621 | gc.collect() # xxx needed? |
|
622 | 622 |
@@ -1,121 +1,121 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """Release data for the IPython project.""" |
|
3 | 3 | |
|
4 | 4 | #***************************************************************************** |
|
5 | 5 | # Copyright (C) 2001-2006 Fernando Perez <fperez@colorado.edu> |
|
6 | 6 | # |
|
7 | 7 | # Copyright (c) 2001 Janko Hauser <jhauser@zscout.de> and Nathaniel Gray |
|
8 | 8 | # <n8gray@caltech.edu> |
|
9 | 9 | # |
|
10 | 10 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | 11 | # the file COPYING, distributed as part of this software. |
|
12 | 12 | #***************************************************************************** |
|
13 | 13 | |
|
14 | 14 | # Name of the package for release purposes. This is the name which labels |
|
15 | 15 | # the tarballs and RPMs made by distutils, so it's best to lowercase it. |
|
16 | 16 | name = 'ipython' |
|
17 | 17 | |
|
18 | 18 | # For versions with substrings (like 0.6.16.svn), use an extra . to separate |
|
19 | 19 | # the new substring. We have to avoid using either dashes or underscores, |
|
20 | 20 | # because bdist_rpm does not accept dashes (an RPM) convention, and |
|
21 | 21 | # bdist_deb does not accept underscores (a Debian convention). |
|
22 | 22 | |
|
23 |
development = |
|
|
24 |
version_base = '0. |
|
|
23 | development = True # change this to False to do a release | |
|
24 | version_base = '0.10' | |
|
25 | 25 | branch = 'ipython' |
|
26 |
revision = '11 |
|
|
26 | revision = '1163' | |
|
27 | 27 | |
|
28 | 28 | if development: |
|
29 | 29 | if branch == 'ipython': |
|
30 | 30 | version = '%s.bzr.r%s' % (version_base, revision) |
|
31 | 31 | else: |
|
32 | 32 | version = '%s.bzr.r%s.%s' % (version_base, revision, branch) |
|
33 | 33 | else: |
|
34 | 34 | version = version_base |
|
35 | 35 | |
|
36 | 36 | |
|
37 | 37 | description = "An interactive computing environment for Python" |
|
38 | 38 | |
|
39 | 39 | long_description = \ |
|
40 | 40 | """ |
|
41 | 41 | The goal of IPython is to create a comprehensive environment for |
|
42 | 42 | interactive and exploratory computing. To support this goal, IPython |
|
43 | 43 | has two main components: |
|
44 | 44 | |
|
45 | 45 | * An enhanced interactive Python shell. |
|
46 | 46 | |
|
47 | 47 | * An architecture for interactive parallel computing. |
|
48 | 48 | |
|
49 | 49 | The enhanced interactive Python shell has the following main features: |
|
50 | 50 | |
|
51 | 51 | * Comprehensive object introspection. |
|
52 | 52 | |
|
53 | 53 | * Input history, persistent across sessions. |
|
54 | 54 | |
|
55 | 55 | * Caching of output results during a session with automatically generated |
|
56 | 56 | references. |
|
57 | 57 | |
|
58 | 58 | * Readline based name completion. |
|
59 | 59 | |
|
60 | 60 | * Extensible system of 'magic' commands for controlling the environment and |
|
61 | 61 | performing many tasks related either to IPython or the operating system. |
|
62 | 62 | |
|
63 | 63 | * Configuration system with easy switching between different setups (simpler |
|
64 | 64 | than changing $PYTHONSTARTUP environment variables every time). |
|
65 | 65 | |
|
66 | 66 | * Session logging and reloading. |
|
67 | 67 | |
|
68 | 68 | * Extensible syntax processing for special purpose situations. |
|
69 | 69 | |
|
70 | 70 | * Access to the system shell with user-extensible alias system. |
|
71 | 71 | |
|
72 | 72 | * Easily embeddable in other Python programs and wxPython GUIs. |
|
73 | 73 | |
|
74 | 74 | * Integrated access to the pdb debugger and the Python profiler. |
|
75 | 75 | |
|
76 | 76 | The parallel computing architecture has the following main features: |
|
77 | 77 | |
|
78 | 78 | * Quickly parallelize Python code from an interactive Python/IPython session. |
|
79 | 79 | |
|
80 | 80 | * A flexible and dynamic process model that be deployed on anything from |
|
81 | 81 | multicore workstations to supercomputers. |
|
82 | 82 | |
|
83 | 83 | * An architecture that supports many different styles of parallelism, from |
|
84 | 84 | message passing to task farming. |
|
85 | 85 | |
|
86 | 86 | * Both blocking and fully asynchronous interfaces. |
|
87 | 87 | |
|
88 | 88 | * High level APIs that enable many things to be parallelized in a few lines |
|
89 | 89 | of code. |
|
90 | 90 | |
|
91 | 91 | * Share live parallel jobs with other users securely. |
|
92 | 92 | |
|
93 | 93 | * Dynamically load balanced task farming system. |
|
94 | 94 | |
|
95 | 95 | * Robust error handling in parallel code. |
|
96 | 96 | |
|
97 | 97 | The latest development version is always available from IPython's `Launchpad |
|
98 | 98 | site <http://launchpad.net/ipython>`_. |
|
99 | 99 | """ |
|
100 | 100 | |
|
101 | 101 | license = 'BSD' |
|
102 | 102 | |
|
103 | 103 | authors = {'Fernando' : ('Fernando Perez','fperez@colorado.edu'), |
|
104 | 104 | 'Janko' : ('Janko Hauser','jhauser@zscout.de'), |
|
105 | 105 | 'Nathan' : ('Nathaniel Gray','n8gray@caltech.edu'), |
|
106 | 106 | 'Ville' : ('Ville Vainio','vivainio@gmail.com'), |
|
107 | 107 | 'Brian' : ('Brian E Granger', 'ellisonbg@gmail.com'), |
|
108 | 108 | 'Min' : ('Min Ragan-Kelley', 'benjaminrk@gmail.com') |
|
109 | 109 | } |
|
110 | 110 | |
|
111 | 111 | author = 'The IPython Development Team' |
|
112 | 112 | |
|
113 | 113 | author_email = 'ipython-dev@scipy.org' |
|
114 | 114 | |
|
115 | 115 | url = 'http://ipython.scipy.org' |
|
116 | 116 | |
|
117 | 117 | download_url = 'http://ipython.scipy.org/dist' |
|
118 | 118 | |
|
119 | 119 | platforms = ['Linux','Mac OSX','Windows XP/2000/NT','Windows 95/98/ME'] |
|
120 | 120 | |
|
121 | 121 | keywords = ['Interactive','Interpreter','Shell','Parallel','Distributed'] |
@@ -1,2790 +1,2865 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """ |
|
3 | 3 | IPython -- An enhanced Interactive Python |
|
4 | 4 | |
|
5 | 5 | Requires Python 2.4 or newer. |
|
6 | 6 | |
|
7 | 7 | This file contains all the classes and helper functions specific to IPython. |
|
8 | 8 | """ |
|
9 | 9 | |
|
10 | 10 | #***************************************************************************** |
|
11 | 11 | # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de> and |
|
12 | 12 | # Copyright (C) 2001-2006 Fernando Perez. <fperez@colorado.edu> |
|
13 | 13 | # |
|
14 | 14 | # Distributed under the terms of the BSD License. The full license is in |
|
15 | 15 | # the file COPYING, distributed as part of this software. |
|
16 | 16 | # |
|
17 | 17 | # Note: this code originally subclassed code.InteractiveConsole from the |
|
18 | 18 | # Python standard library. Over time, all of that class has been copied |
|
19 | 19 | # verbatim here for modifications which could not be accomplished by |
|
20 | 20 | # subclassing. At this point, there are no dependencies at all on the code |
|
21 | 21 | # module anymore (it is not even imported). The Python License (sec. 2) |
|
22 | 22 | # allows for this, but it's always nice to acknowledge credit where credit is |
|
23 | 23 | # due. |
|
24 | 24 | #***************************************************************************** |
|
25 | 25 | |
|
26 | 26 | #**************************************************************************** |
|
27 | 27 | # Modules and globals |
|
28 | 28 | |
|
29 | 29 | # Python standard modules |
|
30 | 30 | import __main__ |
|
31 | 31 | import __builtin__ |
|
32 | 32 | import StringIO |
|
33 | 33 | import bdb |
|
34 | 34 | import cPickle as pickle |
|
35 | 35 | import codeop |
|
36 | 36 | import exceptions |
|
37 | 37 | import glob |
|
38 | 38 | import inspect |
|
39 | 39 | import keyword |
|
40 | 40 | import new |
|
41 | 41 | import os |
|
42 | 42 | import pydoc |
|
43 | 43 | import re |
|
44 | 44 | import shutil |
|
45 | 45 | import string |
|
46 | 46 | import sys |
|
47 | 47 | import tempfile |
|
48 | 48 | import traceback |
|
49 | 49 | import types |
|
50 | 50 | from pprint import pprint, pformat |
|
51 | 51 | |
|
52 | 52 | # IPython's own modules |
|
53 | 53 | #import IPython |
|
54 | 54 | from IPython import Debugger,OInspect,PyColorize,ultraTB |
|
55 | 55 | from IPython.ColorANSI import ColorScheme,ColorSchemeTable # too long names |
|
56 | 56 | from IPython.Extensions import pickleshare |
|
57 | from IPython.FakeModule import FakeModule | |
|
57 | from IPython.FakeModule import FakeModule, init_fakemod_dict | |
|
58 | 58 | from IPython.Itpl import Itpl,itpl,printpl,ItplNS,itplns |
|
59 | 59 | from IPython.Logger import Logger |
|
60 | 60 | from IPython.Magic import Magic |
|
61 | 61 | from IPython.Prompts import CachedOutput |
|
62 | 62 | from IPython.ipstruct import Struct |
|
63 | 63 | from IPython.background_jobs import BackgroundJobManager |
|
64 | 64 | from IPython.usage import cmd_line_usage,interactive_usage |
|
65 | 65 | from IPython.genutils import * |
|
66 | 66 | from IPython.strdispatch import StrDispatch |
|
67 | 67 | import IPython.ipapi |
|
68 | 68 | import IPython.history |
|
69 | 69 | import IPython.prefilter as prefilter |
|
70 | 70 | import IPython.shadowns |
|
71 | 71 | # Globals |
|
72 | 72 | |
|
73 | 73 | # store the builtin raw_input globally, and use this always, in case user code |
|
74 | 74 | # overwrites it (like wx.py.PyShell does) |
|
75 | 75 | raw_input_original = raw_input |
|
76 | 76 | |
|
77 | 77 | # compiled regexps for autoindent management |
|
78 | 78 | dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass') |
|
79 | 79 | |
|
80 | 80 | |
|
81 | 81 | #**************************************************************************** |
|
82 | 82 | # Some utility function definitions |
|
83 | 83 | |
|
84 | 84 | ini_spaces_re = re.compile(r'^(\s+)') |
|
85 | 85 | |
|
86 | 86 | def num_ini_spaces(strng): |
|
87 | 87 | """Return the number of initial spaces in a string""" |
|
88 | 88 | |
|
89 | 89 | ini_spaces = ini_spaces_re.match(strng) |
|
90 | 90 | if ini_spaces: |
|
91 | 91 | return ini_spaces.end() |
|
92 | 92 | else: |
|
93 | 93 | return 0 |
|
94 | 94 | |
|
95 | 95 | def softspace(file, newvalue): |
|
96 | 96 | """Copied from code.py, to remove the dependency""" |
|
97 | 97 | |
|
98 | 98 | oldvalue = 0 |
|
99 | 99 | try: |
|
100 | 100 | oldvalue = file.softspace |
|
101 | 101 | except AttributeError: |
|
102 | 102 | pass |
|
103 | 103 | try: |
|
104 | 104 | file.softspace = newvalue |
|
105 | 105 | except (AttributeError, TypeError): |
|
106 | 106 | # "attribute-less object" or "read-only attributes" |
|
107 | 107 | pass |
|
108 | 108 | return oldvalue |
|
109 | 109 | |
|
110 | 110 | |
|
111 | def user_setup(ipythondir,rc_suffix,mode='install',interactive=True): | |
|
112 | """Install or upgrade the user configuration directory. | |
|
113 | ||
|
114 | Can be called when running for the first time or to upgrade the user's | |
|
115 | .ipython/ directory. | |
|
116 | ||
|
117 | Parameters | |
|
118 | ---------- | |
|
119 | ipythondir : path | |
|
120 | The directory to be used for installation/upgrade. In 'install' mode, | |
|
121 | if this path already exists, the function exits immediately. | |
|
122 | ||
|
123 | rc_suffix : str | |
|
124 | Extension for the config files. On *nix platforms it is typically the | |
|
125 | empty string, while Windows normally uses '.ini'. | |
|
126 | ||
|
127 | mode : str, optional | |
|
128 | Valid modes are 'install' and 'upgrade'. | |
|
129 | ||
|
130 | interactive : bool, optional | |
|
131 | If False, do not wait for user input on any errors. Normally after | |
|
132 | printing its status information, this function waits for the user to | |
|
133 | hit Return before proceeding. This is because the default use case is | |
|
134 | when first installing the IPython configuration, so we want the user to | |
|
135 | acknowledge the initial message, which contains some useful | |
|
136 | information. | |
|
137 | """ | |
|
138 | ||
|
139 | # For automatic use, deactivate all i/o | |
|
140 | if interactive: | |
|
141 | def wait(): | |
|
142 | try: | |
|
143 | raw_input("Please press <RETURN> to start IPython.") | |
|
144 | except EOFError: | |
|
145 | print >> Term.cout | |
|
146 | print '*'*70 | |
|
147 | ||
|
148 | def printf(s): | |
|
149 | print s | |
|
150 | else: | |
|
151 | wait = lambda : None | |
|
152 | printf = lambda s : None | |
|
153 | ||
|
154 | # Install mode should be re-entrant: if the install dir already exists, | |
|
155 | # bail out cleanly | |
|
156 | if mode == 'install' and os.path.isdir(ipythondir): | |
|
157 | return | |
|
158 | ||
|
159 | cwd = os.getcwd() # remember where we started | |
|
160 | glb = glob.glob | |
|
161 | ||
|
162 | printf('*'*70) | |
|
163 | if mode == 'install': | |
|
164 | printf( | |
|
165 | """Welcome to IPython. I will try to create a personal configuration directory | |
|
166 | where you can customize many aspects of IPython's functionality in:\n""") | |
|
167 | else: | |
|
168 | printf('I am going to upgrade your configuration in:') | |
|
169 | ||
|
170 | printf(ipythondir) | |
|
171 | ||
|
172 | rcdirend = os.path.join('IPython','UserConfig') | |
|
173 | cfg = lambda d: os.path.join(d,rcdirend) | |
|
174 | try: | |
|
175 | rcdir = filter(os.path.isdir,map(cfg,sys.path))[0] | |
|
176 | printf("Initializing from configuration: %s" % rcdir) | |
|
177 | except IndexError: | |
|
178 | warning = """ | |
|
179 | Installation error. IPython's directory was not found. | |
|
180 | ||
|
181 | Check the following: | |
|
182 | ||
|
183 | The ipython/IPython directory should be in a directory belonging to your | |
|
184 | PYTHONPATH environment variable (that is, it should be in a directory | |
|
185 | belonging to sys.path). You can copy it explicitly there or just link to it. | |
|
186 | ||
|
187 | IPython will create a minimal default configuration for you. | |
|
188 | ||
|
189 | """ | |
|
190 | warn(warning) | |
|
191 | wait() | |
|
192 | ||
|
193 | if sys.platform =='win32': | |
|
194 | inif = 'ipythonrc.ini' | |
|
195 | else: | |
|
196 | inif = 'ipythonrc' | |
|
197 | minimal_setup = {'ipy_user_conf.py' : 'import ipy_defaults', | |
|
198 | inif : '# intentionally left blank' } | |
|
199 | os.makedirs(ipythondir, mode = 0777) | |
|
200 | for f, cont in minimal_setup.items(): | |
|
201 | # In 2.5, this can be more cleanly done using 'with' | |
|
202 | fobj = file(ipythondir + '/' + f,'w') | |
|
203 | fobj.write(cont) | |
|
204 | fobj.close() | |
|
205 | ||
|
206 | return | |
|
207 | ||
|
208 | if mode == 'install': | |
|
209 | try: | |
|
210 | shutil.copytree(rcdir,ipythondir) | |
|
211 | os.chdir(ipythondir) | |
|
212 | rc_files = glb("ipythonrc*") | |
|
213 | for rc_file in rc_files: | |
|
214 | os.rename(rc_file,rc_file+rc_suffix) | |
|
215 | except: | |
|
216 | warning = """ | |
|
217 | ||
|
218 | There was a problem with the installation: | |
|
219 | %s | |
|
220 | Try to correct it or contact the developers if you think it's a bug. | |
|
221 | IPython will proceed with builtin defaults.""" % sys.exc_info()[1] | |
|
222 | warn(warning) | |
|
223 | wait() | |
|
224 | return | |
|
225 | ||
|
226 | elif mode == 'upgrade': | |
|
227 | try: | |
|
228 | os.chdir(ipythondir) | |
|
229 | except: | |
|
230 | printf(""" | |
|
231 | Can not upgrade: changing to directory %s failed. Details: | |
|
232 | %s | |
|
233 | """ % (ipythondir,sys.exc_info()[1]) ) | |
|
234 | wait() | |
|
235 | return | |
|
236 | else: | |
|
237 | sources = glb(os.path.join(rcdir,'[A-Za-z]*')) | |
|
238 | for new_full_path in sources: | |
|
239 | new_filename = os.path.basename(new_full_path) | |
|
240 | if new_filename.startswith('ipythonrc'): | |
|
241 | new_filename = new_filename + rc_suffix | |
|
242 | # The config directory should only contain files, skip any | |
|
243 | # directories which may be there (like CVS) | |
|
244 | if os.path.isdir(new_full_path): | |
|
245 | continue | |
|
246 | if os.path.exists(new_filename): | |
|
247 | old_file = new_filename+'.old' | |
|
248 | if os.path.exists(old_file): | |
|
249 | os.remove(old_file) | |
|
250 | os.rename(new_filename,old_file) | |
|
251 | shutil.copy(new_full_path,new_filename) | |
|
252 | else: | |
|
253 | raise ValueError('unrecognized mode for install: %r' % mode) | |
|
254 | ||
|
255 | # Fix line-endings to those native to each platform in the config | |
|
256 | # directory. | |
|
257 | try: | |
|
258 | os.chdir(ipythondir) | |
|
259 | except: | |
|
260 | printf(""" | |
|
261 | Problem: changing to directory %s failed. | |
|
262 | Details: | |
|
263 | %s | |
|
264 | ||
|
265 | Some configuration files may have incorrect line endings. This should not | |
|
266 | cause any problems during execution. """ % (ipythondir,sys.exc_info()[1]) ) | |
|
267 | wait() | |
|
268 | else: | |
|
269 | for fname in glb('ipythonrc*'): | |
|
270 | try: | |
|
271 | native_line_ends(fname,backup=0) | |
|
272 | except IOError: | |
|
273 | pass | |
|
274 | ||
|
275 | if mode == 'install': | |
|
276 | printf(""" | |
|
277 | Successful installation! | |
|
278 | ||
|
279 | Please read the sections 'Initial Configuration' and 'Quick Tips' in the | |
|
280 | IPython manual (there are both HTML and PDF versions supplied with the | |
|
281 | distribution) to make sure that your system environment is properly configured | |
|
282 | to take advantage of IPython's features. | |
|
283 | ||
|
284 | Important note: the configuration system has changed! The old system is | |
|
285 | still in place, but its setting may be partly overridden by the settings in | |
|
286 | "~/.ipython/ipy_user_conf.py" config file. Please take a look at the file | |
|
287 | if some of the new settings bother you. | |
|
288 | ||
|
289 | """) | |
|
290 | else: | |
|
291 | printf(""" | |
|
292 | Successful upgrade! | |
|
293 | ||
|
294 | All files in your directory: | |
|
295 | %(ipythondir)s | |
|
296 | which would have been overwritten by the upgrade were backed up with a .old | |
|
297 | extension. If you had made particular customizations in those files you may | |
|
298 | want to merge them back into the new files.""" % locals() ) | |
|
299 | wait() | |
|
300 | os.chdir(cwd) | |
|
301 | ||
|
111 | 302 | #**************************************************************************** |
|
112 | 303 | # Local use exceptions |
|
113 | 304 | class SpaceInInput(exceptions.Exception): pass |
|
114 | 305 | |
|
115 | 306 | |
|
116 | 307 | #**************************************************************************** |
|
117 | 308 | # Local use classes |
|
118 | 309 | class Bunch: pass |
|
119 | 310 | |
|
120 | 311 | class Undefined: pass |
|
121 | 312 | |
|
122 | 313 | class Quitter(object): |
|
123 | 314 | """Simple class to handle exit, similar to Python 2.5's. |
|
124 | 315 | |
|
125 | 316 | It handles exiting in an ipython-safe manner, which the one in Python 2.5 |
|
126 | 317 | doesn't do (obviously, since it doesn't know about ipython).""" |
|
127 | 318 | |
|
128 | 319 | def __init__(self,shell,name): |
|
129 | 320 | self.shell = shell |
|
130 | 321 | self.name = name |
|
131 | 322 | |
|
132 | 323 | def __repr__(self): |
|
133 | 324 | return 'Type %s() to exit.' % self.name |
|
134 | 325 | __str__ = __repr__ |
|
135 | 326 | |
|
136 | 327 | def __call__(self): |
|
137 | 328 | self.shell.exit() |
|
138 | 329 | |
|
139 | 330 | class InputList(list): |
|
140 | 331 | """Class to store user input. |
|
141 | 332 | |
|
142 | 333 | It's basically a list, but slices return a string instead of a list, thus |
|
143 | 334 | allowing things like (assuming 'In' is an instance): |
|
144 | 335 | |
|
145 | 336 | exec In[4:7] |
|
146 | 337 | |
|
147 | 338 | or |
|
148 | 339 | |
|
149 | 340 | exec In[5:9] + In[14] + In[21:25]""" |
|
150 | 341 | |
|
151 | 342 | def __getslice__(self,i,j): |
|
152 | 343 | return ''.join(list.__getslice__(self,i,j)) |
|
153 | 344 | |
|
154 | 345 | class SyntaxTB(ultraTB.ListTB): |
|
155 | 346 | """Extension which holds some state: the last exception value""" |
|
156 | 347 | |
|
157 | 348 | def __init__(self,color_scheme = 'NoColor'): |
|
158 | 349 | ultraTB.ListTB.__init__(self,color_scheme) |
|
159 | 350 | self.last_syntax_error = None |
|
160 | 351 | |
|
161 | 352 | def __call__(self, etype, value, elist): |
|
162 | 353 | self.last_syntax_error = value |
|
163 | 354 | ultraTB.ListTB.__call__(self,etype,value,elist) |
|
164 | 355 | |
|
165 | 356 | def clear_err_state(self): |
|
166 | 357 | """Return the current error state and clear it""" |
|
167 | 358 | e = self.last_syntax_error |
|
168 | 359 | self.last_syntax_error = None |
|
169 | 360 | return e |
|
170 | 361 | |
|
171 | 362 | #**************************************************************************** |
|
172 | 363 | # Main IPython class |
|
173 | 364 | |
|
174 | 365 | # FIXME: the Magic class is a mixin for now, and will unfortunately remain so |
|
175 | 366 | # until a full rewrite is made. I've cleaned all cross-class uses of |
|
176 | 367 | # attributes and methods, but too much user code out there relies on the |
|
177 | 368 | # equlity %foo == __IP.magic_foo, so I can't actually remove the mixin usage. |
|
178 | 369 | # |
|
179 | 370 | # But at least now, all the pieces have been separated and we could, in |
|
180 | 371 | # principle, stop using the mixin. This will ease the transition to the |
|
181 | 372 | # chainsaw branch. |
|
182 | 373 | |
|
183 | 374 | # For reference, the following is the list of 'self.foo' uses in the Magic |
|
184 | 375 | # class as of 2005-12-28. These are names we CAN'T use in the main ipython |
|
185 | 376 | # class, to prevent clashes. |
|
186 | 377 | |
|
187 | 378 | # ['self.__class__', 'self.__dict__', 'self._inspect', 'self._ofind', |
|
188 | 379 | # 'self.arg_err', 'self.extract_input', 'self.format_', 'self.lsmagic', |
|
189 | 380 | # 'self.magic_', 'self.options_table', 'self.parse', 'self.shell', |
|
190 | 381 | # 'self.value'] |
|
191 | 382 | |
|
192 | 383 | class InteractiveShell(object,Magic): |
|
193 | 384 | """An enhanced console for Python.""" |
|
194 | 385 | |
|
195 | 386 | # class attribute to indicate whether the class supports threads or not. |
|
196 | 387 | # Subclasses with thread support should override this as needed. |
|
197 | 388 | isthreaded = False |
|
198 | 389 | |
|
199 | 390 | def __init__(self,name,usage=None,rc=Struct(opts=None,args=None), |
|
200 | 391 | user_ns=None,user_global_ns=None,banner2='', |
|
201 | 392 | custom_exceptions=((),None),embedded=False): |
|
202 | 393 | |
|
203 | 394 | # log system |
|
204 | 395 | self.logger = Logger(self,logfname='ipython_log.py',logmode='rotate') |
|
205 | 396 | |
|
206 | 397 | # Job manager (for jobs run as background threads) |
|
207 | 398 | self.jobs = BackgroundJobManager() |
|
208 | 399 | |
|
209 | 400 | # Store the actual shell's name |
|
210 | 401 | self.name = name |
|
211 | 402 | self.more = False |
|
212 | 403 | |
|
213 | 404 | # We need to know whether the instance is meant for embedding, since |
|
214 | 405 | # global/local namespaces need to be handled differently in that case |
|
215 | 406 | self.embedded = embedded |
|
216 | 407 | if embedded: |
|
217 | 408 | # Control variable so users can, from within the embedded instance, |
|
218 | 409 | # permanently deactivate it. |
|
219 | 410 | self.embedded_active = True |
|
220 | 411 | |
|
221 | 412 | # command compiler |
|
222 | 413 | self.compile = codeop.CommandCompiler() |
|
223 | 414 | |
|
224 | 415 | # User input buffer |
|
225 | 416 | self.buffer = [] |
|
226 | 417 | |
|
227 | 418 | # Default name given in compilation of code |
|
228 | 419 | self.filename = '<ipython console>' |
|
229 | 420 | |
|
230 | 421 | # Install our own quitter instead of the builtins. For python2.3-2.4, |
|
231 | 422 | # this brings in behavior like 2.5, and for 2.5 it's identical. |
|
232 | 423 | __builtin__.exit = Quitter(self,'exit') |
|
233 | 424 | __builtin__.quit = Quitter(self,'quit') |
|
234 | 425 | |
|
235 | 426 | # Make an empty namespace, which extension writers can rely on both |
|
236 | 427 | # existing and NEVER being used by ipython itself. This gives them a |
|
237 | 428 | # convenient location for storing additional information and state |
|
238 | 429 | # their extensions may require, without fear of collisions with other |
|
239 | 430 | # ipython names that may develop later. |
|
240 | 431 | self.meta = Struct() |
|
241 | 432 | |
|
242 | 433 | # Create the namespace where the user will operate. user_ns is |
|
243 | 434 | # normally the only one used, and it is passed to the exec calls as |
|
244 | 435 | # the locals argument. But we do carry a user_global_ns namespace |
|
245 | 436 | # given as the exec 'globals' argument, This is useful in embedding |
|
246 | 437 | # situations where the ipython shell opens in a context where the |
|
247 | 438 | # distinction between locals and globals is meaningful. For |
|
248 | 439 | # non-embedded contexts, it is just the same object as the user_ns dict. |
|
249 | 440 | |
|
250 | 441 | # FIXME. For some strange reason, __builtins__ is showing up at user |
|
251 | 442 | # level as a dict instead of a module. This is a manual fix, but I |
|
252 | 443 | # should really track down where the problem is coming from. Alex |
|
253 | 444 | # Schmolck reported this problem first. |
|
254 | 445 | |
|
255 | 446 | # A useful post by Alex Martelli on this topic: |
|
256 | 447 | # Re: inconsistent value from __builtins__ |
|
257 | 448 | # Von: Alex Martelli <aleaxit@yahoo.com> |
|
258 | 449 | # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends |
|
259 | 450 | # Gruppen: comp.lang.python |
|
260 | 451 | |
|
261 | 452 | # Michael Hohn <hohn@hooknose.lbl.gov> wrote: |
|
262 | 453 | # > >>> print type(builtin_check.get_global_binding('__builtins__')) |
|
263 | 454 | # > <type 'dict'> |
|
264 | 455 | # > >>> print type(__builtins__) |
|
265 | 456 | # > <type 'module'> |
|
266 | 457 | # > Is this difference in return value intentional? |
|
267 | 458 | |
|
268 | 459 | # Well, it's documented that '__builtins__' can be either a dictionary |
|
269 | 460 | # or a module, and it's been that way for a long time. Whether it's |
|
270 | 461 | # intentional (or sensible), I don't know. In any case, the idea is |
|
271 | 462 | # that if you need to access the built-in namespace directly, you |
|
272 | 463 | # should start with "import __builtin__" (note, no 's') which will |
|
273 | 464 | # definitely give you a module. Yeah, it's somewhat confusing:-(. |
|
274 | 465 | |
|
275 | 466 | # These routines return properly built dicts as needed by the rest of |
|
276 | 467 | # the code, and can also be used by extension writers to generate |
|
277 | 468 | # properly initialized namespaces. |
|
278 | 469 | user_ns, user_global_ns = IPython.ipapi.make_user_namespaces(user_ns, |
|
279 | 470 | user_global_ns) |
|
280 | 471 | |
|
281 | 472 | # Assign namespaces |
|
282 | 473 | # This is the namespace where all normal user variables live |
|
283 | 474 | self.user_ns = user_ns |
|
284 | 475 | self.user_global_ns = user_global_ns |
|
285 | 476 | |
|
286 | 477 | # An auxiliary namespace that checks what parts of the user_ns were |
|
287 | 478 | # loaded at startup, so we can list later only variables defined in |
|
288 | 479 | # actual interactive use. Since it is always a subset of user_ns, it |
|
289 | 480 | # doesn't need to be seaparately tracked in the ns_table |
|
290 | 481 | self.user_config_ns = {} |
|
291 | 482 | |
|
292 | 483 | # A namespace to keep track of internal data structures to prevent |
|
293 | 484 | # them from cluttering user-visible stuff. Will be updated later |
|
294 | 485 | self.internal_ns = {} |
|
295 | 486 | |
|
296 | 487 | # Namespace of system aliases. Each entry in the alias |
|
297 | 488 | # table must be a 2-tuple of the form (N,name), where N is the number |
|
298 | 489 | # of positional arguments of the alias. |
|
299 | 490 | self.alias_table = {} |
|
300 | 491 | |
|
301 | 492 | # Now that FakeModule produces a real module, we've run into a nasty |
|
302 | 493 | # problem: after script execution (via %run), the module where the user |
|
303 | 494 | # code ran is deleted. Now that this object is a true module (needed |
|
304 | 495 | # so docetst and other tools work correctly), the Python module |
|
305 | 496 | # teardown mechanism runs over it, and sets to None every variable |
|
306 | 497 | # present in that module. Top-level references to objects from the |
|
307 | 498 | # script survive, because the user_ns is updated with them. However, |
|
308 | 499 | # calling functions defined in the script that use other things from |
|
309 | 500 | # the script will fail, because the function's closure had references |
|
310 | 501 | # to the original objects, which are now all None. So we must protect |
|
311 |
# these modules from deletion by keeping a cache. |
|
|
312 | # stale modules around (we only need the one from the last run), we use | |
|
313 | # a dict keyed with the full path to the script, so only the last | |
|
314 | # version of the module is held in the cache. The %reset command will | |
|
315 | # flush this cache. See the cache_main_mod() and clear_main_mod_cache() | |
|
316 | # methods for details on use. | |
|
317 | self._user_main_modules = {} | |
|
502 | # these modules from deletion by keeping a cache. | |
|
503 | # | |
|
504 | # To avoid keeping stale modules around (we only need the one from the | |
|
505 | # last run), we use a dict keyed with the full path to the script, so | |
|
506 | # only the last version of the module is held in the cache. Note, | |
|
507 | # however, that we must cache the module *namespace contents* (their | |
|
508 | # __dict__). Because if we try to cache the actual modules, old ones | |
|
509 | # (uncached) could be destroyed while still holding references (such as | |
|
510 | # those held by GUI objects that tend to be long-lived)> | |
|
511 | # | |
|
512 | # The %reset command will flush this cache. See the cache_main_mod() | |
|
513 | # and clear_main_mod_cache() methods for details on use. | |
|
514 | ||
|
515 | # This is the cache used for 'main' namespaces | |
|
516 | self._main_ns_cache = {} | |
|
517 | # And this is the single instance of FakeModule whose __dict__ we keep | |
|
518 | # copying and clearing for reuse on each %run | |
|
519 | self._user_main_module = FakeModule() | |
|
318 | 520 | |
|
319 | 521 | # A table holding all the namespaces IPython deals with, so that |
|
320 | 522 | # introspection facilities can search easily. |
|
321 | 523 | self.ns_table = {'user':user_ns, |
|
322 | 524 | 'user_global':user_global_ns, |
|
323 | 525 | 'alias':self.alias_table, |
|
324 | 526 | 'internal':self.internal_ns, |
|
325 | 527 | 'builtin':__builtin__.__dict__ |
|
326 | 528 | } |
|
327 | 529 | |
|
328 | 530 | # Similarly, track all namespaces where references can be held and that |
|
329 | 531 | # we can safely clear (so it can NOT include builtin). This one can be |
|
330 | 532 | # a simple list. |
|
331 | 533 | self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns, |
|
332 | 534 | self.alias_table, self.internal_ns, |
|
333 |
self. |
|
|
535 | self._main_ns_cache ] | |
|
334 | 536 | |
|
335 | 537 | # We need to insert into sys.modules something that looks like a |
|
336 | 538 | # module but which accesses the IPython namespace, for shelve and |
|
337 | 539 | # pickle to work interactively. Normally they rely on getting |
|
338 | 540 | # everything out of __main__, but for embedding purposes each IPython |
|
339 | 541 | # instance has its own private namespace, so we can't go shoving |
|
340 | 542 | # everything into __main__. |
|
341 | 543 | |
|
342 | 544 | # note, however, that we should only do this for non-embedded |
|
343 | 545 | # ipythons, which really mimic the __main__.__dict__ with their own |
|
344 | 546 | # namespace. Embedded instances, on the other hand, should not do |
|
345 | 547 | # this because they need to manage the user local/global namespaces |
|
346 | 548 | # only, but they live within a 'normal' __main__ (meaning, they |
|
347 | 549 | # shouldn't overtake the execution environment of the script they're |
|
348 | 550 | # embedded in). |
|
349 | 551 | |
|
350 | 552 | if not embedded: |
|
351 | 553 | try: |
|
352 | 554 | main_name = self.user_ns['__name__'] |
|
353 | 555 | except KeyError: |
|
354 | 556 | raise KeyError,'user_ns dictionary MUST have a "__name__" key' |
|
355 | 557 | else: |
|
356 | 558 | #print "pickle hack in place" # dbg |
|
357 | 559 | #print 'main_name:',main_name # dbg |
|
358 | 560 | sys.modules[main_name] = FakeModule(self.user_ns) |
|
359 | 561 | |
|
360 | 562 | # List of input with multi-line handling. |
|
361 | 563 | self.input_hist = InputList() |
|
362 | 564 | # This one will hold the 'raw' input history, without any |
|
363 | 565 | # pre-processing. This will allow users to retrieve the input just as |
|
364 | 566 | # it was exactly typed in by the user, with %hist -r. |
|
365 | 567 | self.input_hist_raw = InputList() |
|
366 | 568 | |
|
367 | 569 | # list of visited directories |
|
368 | 570 | try: |
|
369 | 571 | self.dir_hist = [os.getcwd()] |
|
370 | 572 | except OSError: |
|
371 | 573 | self.dir_hist = [] |
|
372 | 574 | |
|
373 | 575 | # dict of output history |
|
374 | 576 | self.output_hist = {} |
|
375 | 577 | |
|
376 | 578 | # Get system encoding at startup time. Certain terminals (like Emacs |
|
377 | 579 | # under Win32 have it set to None, and we need to have a known valid |
|
378 | 580 | # encoding to use in the raw_input() method |
|
379 | 581 | try: |
|
380 | 582 | self.stdin_encoding = sys.stdin.encoding or 'ascii' |
|
381 | 583 | except AttributeError: |
|
382 | 584 | self.stdin_encoding = 'ascii' |
|
383 | 585 | |
|
384 | 586 | # dict of things NOT to alias (keywords, builtins and some magics) |
|
385 | 587 | no_alias = {} |
|
386 | 588 | no_alias_magics = ['cd','popd','pushd','dhist','alias','unalias'] |
|
387 | 589 | for key in keyword.kwlist + no_alias_magics: |
|
388 | 590 | no_alias[key] = 1 |
|
389 | 591 | no_alias.update(__builtin__.__dict__) |
|
390 | 592 | self.no_alias = no_alias |
|
391 | 593 | |
|
392 | 594 | # Object variable to store code object waiting execution. This is |
|
393 | 595 | # used mainly by the multithreaded shells, but it can come in handy in |
|
394 | 596 | # other situations. No need to use a Queue here, since it's a single |
|
395 | 597 | # item which gets cleared once run. |
|
396 | 598 | self.code_to_run = None |
|
397 | 599 | |
|
398 | 600 | # escapes for automatic behavior on the command line |
|
399 | 601 | self.ESC_SHELL = '!' |
|
400 | 602 | self.ESC_SH_CAP = '!!' |
|
401 | 603 | self.ESC_HELP = '?' |
|
402 | 604 | self.ESC_MAGIC = '%' |
|
403 | 605 | self.ESC_QUOTE = ',' |
|
404 | 606 | self.ESC_QUOTE2 = ';' |
|
405 | 607 | self.ESC_PAREN = '/' |
|
406 | 608 | |
|
407 | 609 | # And their associated handlers |
|
408 | 610 | self.esc_handlers = {self.ESC_PAREN : self.handle_auto, |
|
409 | 611 | self.ESC_QUOTE : self.handle_auto, |
|
410 | 612 | self.ESC_QUOTE2 : self.handle_auto, |
|
411 | 613 | self.ESC_MAGIC : self.handle_magic, |
|
412 | 614 | self.ESC_HELP : self.handle_help, |
|
413 | 615 | self.ESC_SHELL : self.handle_shell_escape, |
|
414 | 616 | self.ESC_SH_CAP : self.handle_shell_escape, |
|
415 | 617 | } |
|
416 | 618 | |
|
417 | 619 | # class initializations |
|
418 | 620 | Magic.__init__(self,self) |
|
419 | 621 | |
|
420 | 622 | # Python source parser/formatter for syntax highlighting |
|
421 | 623 | pyformat = PyColorize.Parser().format |
|
422 | 624 | self.pycolorize = lambda src: pyformat(src,'str',self.rc['colors']) |
|
423 | 625 | |
|
424 | 626 | # hooks holds pointers used for user-side customizations |
|
425 | 627 | self.hooks = Struct() |
|
426 | 628 | |
|
427 | 629 | self.strdispatchers = {} |
|
428 | 630 | |
|
429 | 631 | # Set all default hooks, defined in the IPython.hooks module. |
|
430 | 632 | hooks = IPython.hooks |
|
431 | 633 | for hook_name in hooks.__all__: |
|
432 | 634 | # default hooks have priority 100, i.e. low; user hooks should have |
|
433 | 635 | # 0-100 priority |
|
434 | 636 | self.set_hook(hook_name,getattr(hooks,hook_name), 100) |
|
435 | 637 | #print "bound hook",hook_name |
|
436 | 638 | |
|
437 | 639 | # Flag to mark unconditional exit |
|
438 | 640 | self.exit_now = False |
|
439 | 641 | |
|
440 | 642 | self.usage_min = """\ |
|
441 | 643 | An enhanced console for Python. |
|
442 | 644 | Some of its features are: |
|
443 | 645 | - Readline support if the readline library is present. |
|
444 | 646 | - Tab completion in the local namespace. |
|
445 | 647 | - Logging of input, see command-line options. |
|
446 | 648 | - System shell escape via ! , eg !ls. |
|
447 | 649 | - Magic commands, starting with a % (like %ls, %pwd, %cd, etc.) |
|
448 | 650 | - Keeps track of locally defined variables via %who, %whos. |
|
449 | 651 | - Show object information with a ? eg ?x or x? (use ?? for more info). |
|
450 | 652 | """ |
|
451 | 653 | if usage: self.usage = usage |
|
452 | 654 | else: self.usage = self.usage_min |
|
453 | 655 | |
|
454 | 656 | # Storage |
|
455 | 657 | self.rc = rc # This will hold all configuration information |
|
456 | 658 | self.pager = 'less' |
|
457 | 659 | # temporary files used for various purposes. Deleted at exit. |
|
458 | 660 | self.tempfiles = [] |
|
459 | 661 | |
|
460 | 662 | # Keep track of readline usage (later set by init_readline) |
|
461 | 663 | self.has_readline = False |
|
462 | 664 | |
|
463 | 665 | # template for logfile headers. It gets resolved at runtime by the |
|
464 | 666 | # logstart method. |
|
465 | 667 | self.loghead_tpl = \ |
|
466 | 668 | """#log# Automatic Logger file. *** THIS MUST BE THE FIRST LINE *** |
|
467 | 669 | #log# DO NOT CHANGE THIS LINE OR THE TWO BELOW |
|
468 | 670 | #log# opts = %s |
|
469 | 671 | #log# args = %s |
|
470 | 672 | #log# It is safe to make manual edits below here. |
|
471 | 673 | #log#----------------------------------------------------------------------- |
|
472 | 674 | """ |
|
473 | 675 | # for pushd/popd management |
|
474 | 676 | try: |
|
475 | 677 | self.home_dir = get_home_dir() |
|
476 | 678 | except HomeDirError,msg: |
|
477 | 679 | fatal(msg) |
|
478 | 680 | |
|
479 | 681 | self.dir_stack = [] |
|
480 | 682 | |
|
481 | 683 | # Functions to call the underlying shell. |
|
482 | 684 | |
|
483 | 685 | # The first is similar to os.system, but it doesn't return a value, |
|
484 | 686 | # and it allows interpolation of variables in the user's namespace. |
|
485 | 687 | self.system = lambda cmd: \ |
|
486 | 688 | self.hooks.shell_hook(self.var_expand(cmd,depth=2)) |
|
487 | 689 | |
|
488 | 690 | # These are for getoutput and getoutputerror: |
|
489 | 691 | self.getoutput = lambda cmd: \ |
|
490 | 692 | getoutput(self.var_expand(cmd,depth=2), |
|
491 | 693 | header=self.rc.system_header, |
|
492 | 694 | verbose=self.rc.system_verbose) |
|
493 | 695 | |
|
494 | 696 | self.getoutputerror = lambda cmd: \ |
|
495 | 697 | getoutputerror(self.var_expand(cmd,depth=2), |
|
496 | 698 | header=self.rc.system_header, |
|
497 | 699 | verbose=self.rc.system_verbose) |
|
498 | 700 | |
|
499 | 701 | |
|
500 | 702 | # keep track of where we started running (mainly for crash post-mortem) |
|
501 | 703 | self.starting_dir = os.getcwd() |
|
502 | 704 | |
|
503 | 705 | # Various switches which can be set |
|
504 | 706 | self.CACHELENGTH = 5000 # this is cheap, it's just text |
|
505 | 707 | self.BANNER = "Python %(version)s on %(platform)s\n" % sys.__dict__ |
|
506 | 708 | self.banner2 = banner2 |
|
507 | 709 | |
|
508 | 710 | # TraceBack handlers: |
|
509 | 711 | |
|
510 | 712 | # Syntax error handler. |
|
511 | 713 | self.SyntaxTB = SyntaxTB(color_scheme='NoColor') |
|
512 | 714 | |
|
513 | 715 | # The interactive one is initialized with an offset, meaning we always |
|
514 | 716 | # want to remove the topmost item in the traceback, which is our own |
|
515 | 717 | # internal code. Valid modes: ['Plain','Context','Verbose'] |
|
516 | 718 | self.InteractiveTB = ultraTB.AutoFormattedTB(mode = 'Plain', |
|
517 | 719 | color_scheme='NoColor', |
|
518 | 720 | tb_offset = 1) |
|
519 | 721 | |
|
520 | 722 | # IPython itself shouldn't crash. This will produce a detailed |
|
521 | 723 | # post-mortem if it does. But we only install the crash handler for |
|
522 | 724 | # non-threaded shells, the threaded ones use a normal verbose reporter |
|
523 | 725 | # and lose the crash handler. This is because exceptions in the main |
|
524 | 726 | # thread (such as in GUI code) propagate directly to sys.excepthook, |
|
525 | 727 | # and there's no point in printing crash dumps for every user exception. |
|
526 | 728 | if self.isthreaded: |
|
527 | 729 | ipCrashHandler = ultraTB.FormattedTB() |
|
528 | 730 | else: |
|
529 | 731 | from IPython import CrashHandler |
|
530 | 732 | ipCrashHandler = CrashHandler.IPythonCrashHandler(self) |
|
531 | 733 | self.set_crash_handler(ipCrashHandler) |
|
532 | 734 | |
|
533 | 735 | # and add any custom exception handlers the user may have specified |
|
534 | 736 | self.set_custom_exc(*custom_exceptions) |
|
535 | 737 | |
|
536 | 738 | # indentation management |
|
537 | 739 | self.autoindent = False |
|
538 | 740 | self.indent_current_nsp = 0 |
|
539 | 741 | |
|
540 | 742 | # Make some aliases automatically |
|
541 | 743 | # Prepare list of shell aliases to auto-define |
|
542 | 744 | if os.name == 'posix': |
|
543 | 745 | auto_alias = ('mkdir mkdir', 'rmdir rmdir', |
|
544 | 746 | 'mv mv -i','rm rm -i','cp cp -i', |
|
545 | 747 | 'cat cat','less less','clear clear', |
|
546 | 748 | # a better ls |
|
547 | 749 | 'ls ls -F', |
|
548 | 750 | # long ls |
|
549 | 751 | 'll ls -lF') |
|
550 | 752 | # Extra ls aliases with color, which need special treatment on BSD |
|
551 | 753 | # variants |
|
552 | 754 | ls_extra = ( # color ls |
|
553 | 755 | 'lc ls -F -o --color', |
|
554 | 756 | # ls normal files only |
|
555 | 757 | 'lf ls -F -o --color %l | grep ^-', |
|
556 | 758 | # ls symbolic links |
|
557 | 759 | 'lk ls -F -o --color %l | grep ^l', |
|
558 | 760 | # directories or links to directories, |
|
559 | 761 | 'ldir ls -F -o --color %l | grep /$', |
|
560 | 762 | # things which are executable |
|
561 | 763 | 'lx ls -F -o --color %l | grep ^-..x', |
|
562 | 764 | ) |
|
563 | 765 | # The BSDs don't ship GNU ls, so they don't understand the |
|
564 | 766 | # --color switch out of the box |
|
565 | 767 | if 'bsd' in sys.platform: |
|
566 | 768 | ls_extra = ( # ls normal files only |
|
567 | 769 | 'lf ls -lF | grep ^-', |
|
568 | 770 | # ls symbolic links |
|
569 | 771 | 'lk ls -lF | grep ^l', |
|
570 | 772 | # directories or links to directories, |
|
571 | 773 | 'ldir ls -lF | grep /$', |
|
572 | 774 | # things which are executable |
|
573 | 775 | 'lx ls -lF | grep ^-..x', |
|
574 | 776 | ) |
|
575 | 777 | auto_alias = auto_alias + ls_extra |
|
576 | 778 | elif os.name in ['nt','dos']: |
|
577 | 779 | auto_alias = ('ls dir /on', |
|
578 | 780 | 'ddir dir /ad /on', 'ldir dir /ad /on', |
|
579 | 781 | 'mkdir mkdir','rmdir rmdir','echo echo', |
|
580 | 782 | 'ren ren','cls cls','copy copy') |
|
581 | 783 | else: |
|
582 | 784 | auto_alias = () |
|
583 | 785 | self.auto_alias = [s.split(None,1) for s in auto_alias] |
|
584 | 786 | |
|
585 | 787 | # Produce a public API instance |
|
586 | 788 | self.api = IPython.ipapi.IPApi(self) |
|
587 | 789 | |
|
588 | 790 | # Initialize all user-visible namespaces |
|
589 | 791 | self.init_namespaces() |
|
590 | 792 | |
|
591 | 793 | # Call the actual (public) initializer |
|
592 | 794 | self.init_auto_alias() |
|
593 | 795 | |
|
594 | 796 | # track which builtins we add, so we can clean up later |
|
595 | 797 | self.builtins_added = {} |
|
596 | 798 | # This method will add the necessary builtins for operation, but |
|
597 | 799 | # tracking what it did via the builtins_added dict. |
|
598 | 800 | |
|
599 | 801 | #TODO: remove this, redundant |
|
600 | 802 | self.add_builtins() |
|
601 | 803 | # end __init__ |
|
602 | 804 | |
|
603 | 805 | def var_expand(self,cmd,depth=0): |
|
604 | 806 | """Expand python variables in a string. |
|
605 | 807 | |
|
606 | 808 | The depth argument indicates how many frames above the caller should |
|
607 | 809 | be walked to look for the local namespace where to expand variables. |
|
608 | 810 | |
|
609 | 811 | The global namespace for expansion is always the user's interactive |
|
610 | 812 | namespace. |
|
611 | 813 | """ |
|
612 | 814 | |
|
613 | 815 | return str(ItplNS(cmd, |
|
614 | 816 | self.user_ns, # globals |
|
615 | 817 | # Skip our own frame in searching for locals: |
|
616 | 818 | sys._getframe(depth+1).f_locals # locals |
|
617 | 819 | )) |
|
618 | 820 | |
|
619 | 821 | def pre_config_initialization(self): |
|
620 | 822 | """Pre-configuration init method |
|
621 | 823 | |
|
622 | 824 | This is called before the configuration files are processed to |
|
623 | 825 | prepare the services the config files might need. |
|
624 | 826 | |
|
625 | 827 | self.rc already has reasonable default values at this point. |
|
626 | 828 | """ |
|
627 | 829 | rc = self.rc |
|
628 | 830 | try: |
|
629 | 831 | self.db = pickleshare.PickleShareDB(rc.ipythondir + "/db") |
|
630 | 832 | except exceptions.UnicodeDecodeError: |
|
631 | 833 | print "Your ipythondir can't be decoded to unicode!" |
|
632 | 834 | print "Please set HOME environment variable to something that" |
|
633 | 835 | print r"only has ASCII characters, e.g. c:\home" |
|
634 | 836 | print "Now it is",rc.ipythondir |
|
635 | 837 | sys.exit() |
|
636 | 838 | self.shadowhist = IPython.history.ShadowHist(self.db) |
|
637 | 839 | |
|
638 | 840 | def post_config_initialization(self): |
|
639 | 841 | """Post configuration init method |
|
640 | 842 | |
|
641 | 843 | This is called after the configuration files have been processed to |
|
642 | 844 | 'finalize' the initialization.""" |
|
643 | 845 | |
|
644 | 846 | rc = self.rc |
|
645 | 847 | |
|
646 | 848 | # Object inspector |
|
647 | 849 | self.inspector = OInspect.Inspector(OInspect.InspectColors, |
|
648 | 850 | PyColorize.ANSICodeColors, |
|
649 | 851 | 'NoColor', |
|
650 | 852 | rc.object_info_string_level) |
|
651 | 853 | |
|
652 | 854 | self.rl_next_input = None |
|
653 | 855 | self.rl_do_indent = False |
|
654 | 856 | # Load readline proper |
|
655 | 857 | if rc.readline: |
|
656 | 858 | self.init_readline() |
|
657 | 859 | |
|
658 | 860 | # local shortcut, this is used a LOT |
|
659 | 861 | self.log = self.logger.log |
|
660 | 862 | |
|
661 | 863 | # Initialize cache, set in/out prompts and printing system |
|
662 | 864 | self.outputcache = CachedOutput(self, |
|
663 | 865 | rc.cache_size, |
|
664 | 866 | rc.pprint, |
|
665 | 867 | input_sep = rc.separate_in, |
|
666 | 868 | output_sep = rc.separate_out, |
|
667 | 869 | output_sep2 = rc.separate_out2, |
|
668 | 870 | ps1 = rc.prompt_in1, |
|
669 | 871 | ps2 = rc.prompt_in2, |
|
670 | 872 | ps_out = rc.prompt_out, |
|
671 | 873 | pad_left = rc.prompts_pad_left) |
|
672 | 874 | |
|
673 | 875 | # user may have over-ridden the default print hook: |
|
674 | 876 | try: |
|
675 | 877 | self.outputcache.__class__.display = self.hooks.display |
|
676 | 878 | except AttributeError: |
|
677 | 879 | pass |
|
678 | 880 | |
|
679 | 881 | # I don't like assigning globally to sys, because it means when |
|
680 | 882 | # embedding instances, each embedded instance overrides the previous |
|
681 | 883 | # choice. But sys.displayhook seems to be called internally by exec, |
|
682 | 884 | # so I don't see a way around it. We first save the original and then |
|
683 | 885 | # overwrite it. |
|
684 | 886 | self.sys_displayhook = sys.displayhook |
|
685 | 887 | sys.displayhook = self.outputcache |
|
686 | 888 | |
|
687 | 889 | # Do a proper resetting of doctest, including the necessary displayhook |
|
688 | 890 | # monkeypatching |
|
689 | 891 | try: |
|
690 | 892 | doctest_reload() |
|
691 | 893 | except ImportError: |
|
692 | 894 | warn("doctest module does not exist.") |
|
693 | 895 | |
|
694 | 896 | # Set user colors (don't do it in the constructor above so that it |
|
695 | 897 | # doesn't crash if colors option is invalid) |
|
696 | 898 | self.magic_colors(rc.colors) |
|
697 | 899 | |
|
698 | 900 | # Set calling of pdb on exceptions |
|
699 | 901 | self.call_pdb = rc.pdb |
|
700 | 902 | |
|
701 | 903 | # Load user aliases |
|
702 | 904 | for alias in rc.alias: |
|
703 | 905 | self.magic_alias(alias) |
|
704 | 906 | |
|
705 | 907 | self.hooks.late_startup_hook() |
|
706 | 908 | |
|
707 | 909 | for cmd in self.rc.autoexec: |
|
708 | 910 | #print "autoexec>",cmd #dbg |
|
709 | 911 | self.api.runlines(cmd) |
|
710 | 912 | |
|
711 | 913 | batchrun = False |
|
712 | 914 | for batchfile in [path(arg) for arg in self.rc.args |
|
713 | 915 | if arg.lower().endswith('.ipy')]: |
|
714 | 916 | if not batchfile.isfile(): |
|
715 | 917 | print "No such batch file:", batchfile |
|
716 | 918 | continue |
|
717 | 919 | self.api.runlines(batchfile.text()) |
|
718 | 920 | batchrun = True |
|
719 | 921 | # without -i option, exit after running the batch file |
|
720 | 922 | if batchrun and not self.rc.interact: |
|
721 | 923 | self.ask_exit() |
|
722 | 924 | |
|
723 | 925 | def init_namespaces(self): |
|
724 | 926 | """Initialize all user-visible namespaces to their minimum defaults. |
|
725 | 927 | |
|
726 | 928 | Certain history lists are also initialized here, as they effectively |
|
727 | 929 | act as user namespaces. |
|
728 | 930 | |
|
729 | 931 | Note |
|
730 | 932 | ---- |
|
731 | 933 | All data structures here are only filled in, they are NOT reset by this |
|
732 | 934 | method. If they were not empty before, data will simply be added to |
|
733 | 935 | therm. |
|
734 | 936 | """ |
|
735 | 937 | # The user namespace MUST have a pointer to the shell itself. |
|
736 | 938 | self.user_ns[self.name] = self |
|
737 | 939 | |
|
738 | 940 | # Store the public api instance |
|
739 | 941 | self.user_ns['_ip'] = self.api |
|
740 | 942 | |
|
741 | 943 | # make global variables for user access to the histories |
|
742 | 944 | self.user_ns['_ih'] = self.input_hist |
|
743 | 945 | self.user_ns['_oh'] = self.output_hist |
|
744 | 946 | self.user_ns['_dh'] = self.dir_hist |
|
745 | 947 | |
|
746 | 948 | # user aliases to input and output histories |
|
747 | 949 | self.user_ns['In'] = self.input_hist |
|
748 | 950 | self.user_ns['Out'] = self.output_hist |
|
749 | 951 | |
|
750 | 952 | self.user_ns['_sh'] = IPython.shadowns |
|
751 | 953 | |
|
752 | 954 | # Fill the history zero entry, user counter starts at 1 |
|
753 | 955 | self.input_hist.append('\n') |
|
754 | 956 | self.input_hist_raw.append('\n') |
|
755 | 957 | |
|
756 | 958 | def add_builtins(self): |
|
757 | 959 | """Store ipython references into the builtin namespace. |
|
758 | 960 | |
|
759 | 961 | Some parts of ipython operate via builtins injected here, which hold a |
|
760 | 962 | reference to IPython itself.""" |
|
761 | 963 | |
|
762 | 964 | # TODO: deprecate all of these, they are unsafe |
|
763 | 965 | builtins_new = dict(__IPYTHON__ = self, |
|
764 | 966 | ip_set_hook = self.set_hook, |
|
765 | 967 | jobs = self.jobs, |
|
766 | 968 | ipmagic = wrap_deprecated(self.ipmagic,'_ip.magic()'), |
|
767 | 969 | ipalias = wrap_deprecated(self.ipalias), |
|
768 | 970 | ipsystem = wrap_deprecated(self.ipsystem,'_ip.system()'), |
|
769 | 971 | #_ip = self.api |
|
770 | 972 | ) |
|
771 | 973 | for biname,bival in builtins_new.items(): |
|
772 | 974 | try: |
|
773 | 975 | # store the orignal value so we can restore it |
|
774 | 976 | self.builtins_added[biname] = __builtin__.__dict__[biname] |
|
775 | 977 | except KeyError: |
|
776 | 978 | # or mark that it wasn't defined, and we'll just delete it at |
|
777 | 979 | # cleanup |
|
778 | 980 | self.builtins_added[biname] = Undefined |
|
779 | 981 | __builtin__.__dict__[biname] = bival |
|
780 | 982 | |
|
781 | 983 | # Keep in the builtins a flag for when IPython is active. We set it |
|
782 | 984 | # with setdefault so that multiple nested IPythons don't clobber one |
|
783 | 985 | # another. Each will increase its value by one upon being activated, |
|
784 | 986 | # which also gives us a way to determine the nesting level. |
|
785 | 987 | __builtin__.__dict__.setdefault('__IPYTHON__active',0) |
|
786 | 988 | |
|
787 | 989 | def clean_builtins(self): |
|
788 | 990 | """Remove any builtins which might have been added by add_builtins, or |
|
789 | 991 | restore overwritten ones to their previous values.""" |
|
790 | 992 | for biname,bival in self.builtins_added.items(): |
|
791 | 993 | if bival is Undefined: |
|
792 | 994 | del __builtin__.__dict__[biname] |
|
793 | 995 | else: |
|
794 | 996 | __builtin__.__dict__[biname] = bival |
|
795 | 997 | self.builtins_added.clear() |
|
796 | 998 | |
|
797 | 999 | def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None): |
|
798 | 1000 | """set_hook(name,hook) -> sets an internal IPython hook. |
|
799 | 1001 | |
|
800 | 1002 | IPython exposes some of its internal API as user-modifiable hooks. By |
|
801 | 1003 | adding your function to one of these hooks, you can modify IPython's |
|
802 | 1004 | behavior to call at runtime your own routines.""" |
|
803 | 1005 | |
|
804 | 1006 | # At some point in the future, this should validate the hook before it |
|
805 | 1007 | # accepts it. Probably at least check that the hook takes the number |
|
806 | 1008 | # of args it's supposed to. |
|
807 | 1009 | |
|
808 | 1010 | f = new.instancemethod(hook,self,self.__class__) |
|
809 | 1011 | |
|
810 | 1012 | # check if the hook is for strdispatcher first |
|
811 | 1013 | if str_key is not None: |
|
812 | 1014 | sdp = self.strdispatchers.get(name, StrDispatch()) |
|
813 | 1015 | sdp.add_s(str_key, f, priority ) |
|
814 | 1016 | self.strdispatchers[name] = sdp |
|
815 | 1017 | return |
|
816 | 1018 | if re_key is not None: |
|
817 | 1019 | sdp = self.strdispatchers.get(name, StrDispatch()) |
|
818 | 1020 | sdp.add_re(re.compile(re_key), f, priority ) |
|
819 | 1021 | self.strdispatchers[name] = sdp |
|
820 | 1022 | return |
|
821 | 1023 | |
|
822 | 1024 | dp = getattr(self.hooks, name, None) |
|
823 | 1025 | if name not in IPython.hooks.__all__: |
|
824 | 1026 | print "Warning! Hook '%s' is not one of %s" % (name, IPython.hooks.__all__ ) |
|
825 | 1027 | if not dp: |
|
826 | 1028 | dp = IPython.hooks.CommandChainDispatcher() |
|
827 | 1029 | |
|
828 | 1030 | try: |
|
829 | 1031 | dp.add(f,priority) |
|
830 | 1032 | except AttributeError: |
|
831 | 1033 | # it was not commandchain, plain old func - replace |
|
832 | 1034 | dp = f |
|
833 | 1035 | |
|
834 | 1036 | setattr(self.hooks,name, dp) |
|
835 | 1037 | |
|
836 | 1038 | |
|
837 | 1039 | #setattr(self.hooks,name,new.instancemethod(hook,self,self.__class__)) |
|
838 | 1040 | |
|
839 | 1041 | def set_crash_handler(self,crashHandler): |
|
840 | 1042 | """Set the IPython crash handler. |
|
841 | 1043 | |
|
842 | 1044 | This must be a callable with a signature suitable for use as |
|
843 | 1045 | sys.excepthook.""" |
|
844 | 1046 | |
|
845 | 1047 | # Install the given crash handler as the Python exception hook |
|
846 | 1048 | sys.excepthook = crashHandler |
|
847 | 1049 | |
|
848 | 1050 | # The instance will store a pointer to this, so that runtime code |
|
849 | 1051 | # (such as magics) can access it. This is because during the |
|
850 | 1052 | # read-eval loop, it gets temporarily overwritten (to deal with GUI |
|
851 | 1053 | # frameworks). |
|
852 | 1054 | self.sys_excepthook = sys.excepthook |
|
853 | 1055 | |
|
854 | 1056 | |
|
855 | 1057 | def set_custom_exc(self,exc_tuple,handler): |
|
856 | 1058 | """set_custom_exc(exc_tuple,handler) |
|
857 | 1059 | |
|
858 | 1060 | Set a custom exception handler, which will be called if any of the |
|
859 | 1061 | exceptions in exc_tuple occur in the mainloop (specifically, in the |
|
860 | 1062 | runcode() method. |
|
861 | 1063 | |
|
862 | 1064 | Inputs: |
|
863 | 1065 | |
|
864 | 1066 | - exc_tuple: a *tuple* of valid exceptions to call the defined |
|
865 | 1067 | handler for. It is very important that you use a tuple, and NOT A |
|
866 | 1068 | LIST here, because of the way Python's except statement works. If |
|
867 | 1069 | you only want to trap a single exception, use a singleton tuple: |
|
868 | 1070 | |
|
869 | 1071 | exc_tuple == (MyCustomException,) |
|
870 | 1072 | |
|
871 | 1073 | - handler: this must be defined as a function with the following |
|
872 | 1074 | basic interface: def my_handler(self,etype,value,tb). |
|
873 | 1075 | |
|
874 | 1076 | This will be made into an instance method (via new.instancemethod) |
|
875 | 1077 | of IPython itself, and it will be called if any of the exceptions |
|
876 | 1078 | listed in the exc_tuple are caught. If the handler is None, an |
|
877 | 1079 | internal basic one is used, which just prints basic info. |
|
878 | 1080 | |
|
879 | 1081 | WARNING: by putting in your own exception handler into IPython's main |
|
880 | 1082 | execution loop, you run a very good chance of nasty crashes. This |
|
881 | 1083 | facility should only be used if you really know what you are doing.""" |
|
882 | 1084 | |
|
883 | 1085 | assert type(exc_tuple)==type(()) , \ |
|
884 | 1086 | "The custom exceptions must be given AS A TUPLE." |
|
885 | 1087 | |
|
886 | 1088 | def dummy_handler(self,etype,value,tb): |
|
887 | 1089 | print '*** Simple custom exception handler ***' |
|
888 | 1090 | print 'Exception type :',etype |
|
889 | 1091 | print 'Exception value:',value |
|
890 | 1092 | print 'Traceback :',tb |
|
891 | 1093 | print 'Source code :','\n'.join(self.buffer) |
|
892 | 1094 | |
|
893 | 1095 | if handler is None: handler = dummy_handler |
|
894 | 1096 | |
|
895 | 1097 | self.CustomTB = new.instancemethod(handler,self,self.__class__) |
|
896 | 1098 | self.custom_exceptions = exc_tuple |
|
897 | 1099 | |
|
898 | 1100 | def set_custom_completer(self,completer,pos=0): |
|
899 | 1101 | """set_custom_completer(completer,pos=0) |
|
900 | 1102 | |
|
901 | 1103 | Adds a new custom completer function. |
|
902 | 1104 | |
|
903 | 1105 | The position argument (defaults to 0) is the index in the completers |
|
904 | 1106 | list where you want the completer to be inserted.""" |
|
905 | 1107 | |
|
906 | 1108 | newcomp = new.instancemethod(completer,self.Completer, |
|
907 | 1109 | self.Completer.__class__) |
|
908 | 1110 | self.Completer.matchers.insert(pos,newcomp) |
|
909 | 1111 | |
|
910 | 1112 | def set_completer(self): |
|
911 | 1113 | """reset readline's completer to be our own.""" |
|
912 | 1114 | self.readline.set_completer(self.Completer.complete) |
|
913 | 1115 | |
|
914 | 1116 | def _get_call_pdb(self): |
|
915 | 1117 | return self._call_pdb |
|
916 | 1118 | |
|
917 | 1119 | def _set_call_pdb(self,val): |
|
918 | 1120 | |
|
919 | 1121 | if val not in (0,1,False,True): |
|
920 | 1122 | raise ValueError,'new call_pdb value must be boolean' |
|
921 | 1123 | |
|
922 | 1124 | # store value in instance |
|
923 | 1125 | self._call_pdb = val |
|
924 | 1126 | |
|
925 | 1127 | # notify the actual exception handlers |
|
926 | 1128 | self.InteractiveTB.call_pdb = val |
|
927 | 1129 | if self.isthreaded: |
|
928 | 1130 | try: |
|
929 | 1131 | self.sys_excepthook.call_pdb = val |
|
930 | 1132 | except: |
|
931 | 1133 | warn('Failed to activate pdb for threaded exception handler') |
|
932 | 1134 | |
|
933 | 1135 | call_pdb = property(_get_call_pdb,_set_call_pdb,None, |
|
934 | 1136 | 'Control auto-activation of pdb at exceptions') |
|
935 | 1137 | |
|
936 | 1138 | # These special functions get installed in the builtin namespace, to |
|
937 | 1139 | # provide programmatic (pure python) access to magics, aliases and system |
|
938 | 1140 | # calls. This is important for logging, user scripting, and more. |
|
939 | 1141 | |
|
940 | 1142 | # We are basically exposing, via normal python functions, the three |
|
941 | 1143 | # mechanisms in which ipython offers special call modes (magics for |
|
942 | 1144 | # internal control, aliases for direct system access via pre-selected |
|
943 | 1145 | # names, and !cmd for calling arbitrary system commands). |
|
944 | 1146 | |
|
945 | 1147 | def ipmagic(self,arg_s): |
|
946 | 1148 | """Call a magic function by name. |
|
947 | 1149 | |
|
948 | 1150 | Input: a string containing the name of the magic function to call and any |
|
949 | 1151 | additional arguments to be passed to the magic. |
|
950 | 1152 | |
|
951 | 1153 | ipmagic('name -opt foo bar') is equivalent to typing at the ipython |
|
952 | 1154 | prompt: |
|
953 | 1155 | |
|
954 | 1156 | In[1]: %name -opt foo bar |
|
955 | 1157 | |
|
956 | 1158 | To call a magic without arguments, simply use ipmagic('name'). |
|
957 | 1159 | |
|
958 | 1160 | This provides a proper Python function to call IPython's magics in any |
|
959 | 1161 | valid Python code you can type at the interpreter, including loops and |
|
960 | 1162 | compound statements. It is added by IPython to the Python builtin |
|
961 | 1163 | namespace upon initialization.""" |
|
962 | 1164 | |
|
963 | 1165 | args = arg_s.split(' ',1) |
|
964 | 1166 | magic_name = args[0] |
|
965 | 1167 | magic_name = magic_name.lstrip(self.ESC_MAGIC) |
|
966 | 1168 | |
|
967 | 1169 | try: |
|
968 | 1170 | magic_args = args[1] |
|
969 | 1171 | except IndexError: |
|
970 | 1172 | magic_args = '' |
|
971 | 1173 | fn = getattr(self,'magic_'+magic_name,None) |
|
972 | 1174 | if fn is None: |
|
973 | 1175 | error("Magic function `%s` not found." % magic_name) |
|
974 | 1176 | else: |
|
975 | 1177 | magic_args = self.var_expand(magic_args,1) |
|
976 | 1178 | return fn(magic_args) |
|
977 | 1179 | |
|
978 | 1180 | def ipalias(self,arg_s): |
|
979 | 1181 | """Call an alias by name. |
|
980 | 1182 | |
|
981 | 1183 | Input: a string containing the name of the alias to call and any |
|
982 | 1184 | additional arguments to be passed to the magic. |
|
983 | 1185 | |
|
984 | 1186 | ipalias('name -opt foo bar') is equivalent to typing at the ipython |
|
985 | 1187 | prompt: |
|
986 | 1188 | |
|
987 | 1189 | In[1]: name -opt foo bar |
|
988 | 1190 | |
|
989 | 1191 | To call an alias without arguments, simply use ipalias('name'). |
|
990 | 1192 | |
|
991 | 1193 | This provides a proper Python function to call IPython's aliases in any |
|
992 | 1194 | valid Python code you can type at the interpreter, including loops and |
|
993 | 1195 | compound statements. It is added by IPython to the Python builtin |
|
994 | 1196 | namespace upon initialization.""" |
|
995 | 1197 | |
|
996 | 1198 | args = arg_s.split(' ',1) |
|
997 | 1199 | alias_name = args[0] |
|
998 | 1200 | try: |
|
999 | 1201 | alias_args = args[1] |
|
1000 | 1202 | except IndexError: |
|
1001 | 1203 | alias_args = '' |
|
1002 | 1204 | if alias_name in self.alias_table: |
|
1003 | 1205 | self.call_alias(alias_name,alias_args) |
|
1004 | 1206 | else: |
|
1005 | 1207 | error("Alias `%s` not found." % alias_name) |
|
1006 | 1208 | |
|
1007 | 1209 | def ipsystem(self,arg_s): |
|
1008 | 1210 | """Make a system call, using IPython.""" |
|
1009 | 1211 | |
|
1010 | 1212 | self.system(arg_s) |
|
1011 | 1213 | |
|
1012 | 1214 | def complete(self,text): |
|
1013 | 1215 | """Return a sorted list of all possible completions on text. |
|
1014 | 1216 | |
|
1015 | 1217 | Inputs: |
|
1016 | 1218 | |
|
1017 | 1219 | - text: a string of text to be completed on. |
|
1018 | 1220 | |
|
1019 | 1221 | This is a wrapper around the completion mechanism, similar to what |
|
1020 | 1222 | readline does at the command line when the TAB key is hit. By |
|
1021 | 1223 | exposing it as a method, it can be used by other non-readline |
|
1022 | 1224 | environments (such as GUIs) for text completion. |
|
1023 | 1225 | |
|
1024 | 1226 | Simple usage example: |
|
1025 | 1227 | |
|
1026 | 1228 | In [7]: x = 'hello' |
|
1027 | 1229 | |
|
1028 | 1230 | In [8]: x |
|
1029 | 1231 | Out[8]: 'hello' |
|
1030 | 1232 | |
|
1031 | 1233 | In [9]: print x |
|
1032 | 1234 | hello |
|
1033 | 1235 | |
|
1034 | 1236 | In [10]: _ip.IP.complete('x.l') |
|
1035 | 1237 | Out[10]: ['x.ljust', 'x.lower', 'x.lstrip'] |
|
1036 | 1238 | """ |
|
1037 | 1239 | |
|
1038 | 1240 | complete = self.Completer.complete |
|
1039 | 1241 | state = 0 |
|
1040 | 1242 | # use a dict so we get unique keys, since ipyhton's multiple |
|
1041 | 1243 | # completers can return duplicates. When we make 2.4 a requirement, |
|
1042 | 1244 | # start using sets instead, which are faster. |
|
1043 | 1245 | comps = {} |
|
1044 | 1246 | while True: |
|
1045 | 1247 | newcomp = complete(text,state,line_buffer=text) |
|
1046 | 1248 | if newcomp is None: |
|
1047 | 1249 | break |
|
1048 | 1250 | comps[newcomp] = 1 |
|
1049 | 1251 | state += 1 |
|
1050 | 1252 | outcomps = comps.keys() |
|
1051 | 1253 | outcomps.sort() |
|
1052 | 1254 | #print "T:",text,"OC:",outcomps # dbg |
|
1053 | 1255 | #print "vars:",self.user_ns.keys() |
|
1054 | 1256 | return outcomps |
|
1055 | 1257 | |
|
1056 | 1258 | def set_completer_frame(self, frame=None): |
|
1057 | 1259 | if frame: |
|
1058 | 1260 | self.Completer.namespace = frame.f_locals |
|
1059 | 1261 | self.Completer.global_namespace = frame.f_globals |
|
1060 | 1262 | else: |
|
1061 | 1263 | self.Completer.namespace = self.user_ns |
|
1062 | 1264 | self.Completer.global_namespace = self.user_global_ns |
|
1063 | 1265 | |
|
1064 | 1266 | def init_auto_alias(self): |
|
1065 | 1267 | """Define some aliases automatically. |
|
1066 | 1268 | |
|
1067 | 1269 | These are ALL parameter-less aliases""" |
|
1068 | 1270 | |
|
1069 | 1271 | for alias,cmd in self.auto_alias: |
|
1070 | 1272 | self.getapi().defalias(alias,cmd) |
|
1071 | 1273 | |
|
1072 | 1274 | |
|
1073 | 1275 | def alias_table_validate(self,verbose=0): |
|
1074 | 1276 | """Update information about the alias table. |
|
1075 | 1277 | |
|
1076 | 1278 | In particular, make sure no Python keywords/builtins are in it.""" |
|
1077 | 1279 | |
|
1078 | 1280 | no_alias = self.no_alias |
|
1079 | 1281 | for k in self.alias_table.keys(): |
|
1080 | 1282 | if k in no_alias: |
|
1081 | 1283 | del self.alias_table[k] |
|
1082 | 1284 | if verbose: |
|
1083 | 1285 | print ("Deleting alias <%s>, it's a Python " |
|
1084 | 1286 | "keyword or builtin." % k) |
|
1085 | 1287 | |
|
1086 | 1288 | def set_autoindent(self,value=None): |
|
1087 | 1289 | """Set the autoindent flag, checking for readline support. |
|
1088 | 1290 | |
|
1089 | 1291 | If called with no arguments, it acts as a toggle.""" |
|
1090 | 1292 | |
|
1091 | 1293 | if not self.has_readline: |
|
1092 | 1294 | if os.name == 'posix': |
|
1093 | 1295 | warn("The auto-indent feature requires the readline library") |
|
1094 | 1296 | self.autoindent = 0 |
|
1095 | 1297 | return |
|
1096 | 1298 | if value is None: |
|
1097 | 1299 | self.autoindent = not self.autoindent |
|
1098 | 1300 | else: |
|
1099 | 1301 | self.autoindent = value |
|
1100 | 1302 | |
|
1101 | 1303 | def rc_set_toggle(self,rc_field,value=None): |
|
1102 | 1304 | """Set or toggle a field in IPython's rc config. structure. |
|
1103 | 1305 | |
|
1104 | 1306 | If called with no arguments, it acts as a toggle. |
|
1105 | 1307 | |
|
1106 | 1308 | If called with a non-existent field, the resulting AttributeError |
|
1107 | 1309 | exception will propagate out.""" |
|
1108 | 1310 | |
|
1109 | 1311 | rc_val = getattr(self.rc,rc_field) |
|
1110 | 1312 | if value is None: |
|
1111 | 1313 | value = not rc_val |
|
1112 | 1314 | setattr(self.rc,rc_field,value) |
|
1113 | 1315 | |
|
1114 | 1316 | def user_setup(self,ipythondir,rc_suffix,mode='install'): |
|
1115 | 1317 | """Install the user configuration directory. |
|
1116 | 1318 | |
|
1117 | Can be called when running for the first time or to upgrade the user's | |
|
1118 | .ipython/ directory with the mode parameter. Valid modes are 'install' | |
|
1119 | and 'upgrade'.""" | |
|
1120 | ||
|
1121 | def wait(): | |
|
1122 | try: | |
|
1123 | raw_input("Please press <RETURN> to start IPython.") | |
|
1124 | except EOFError: | |
|
1125 | print >> Term.cout | |
|
1126 | print '*'*70 | |
|
1127 | ||
|
1128 | cwd = os.getcwd() # remember where we started | |
|
1129 | glb = glob.glob | |
|
1130 | print '*'*70 | |
|
1131 | if mode == 'install': | |
|
1132 | print \ | |
|
1133 | """Welcome to IPython. I will try to create a personal configuration directory | |
|
1134 | where you can customize many aspects of IPython's functionality in:\n""" | |
|
1135 | else: | |
|
1136 | print 'I am going to upgrade your configuration in:' | |
|
1137 | ||
|
1138 | print ipythondir | |
|
1139 | ||
|
1140 | rcdirend = os.path.join('IPython','UserConfig') | |
|
1141 | cfg = lambda d: os.path.join(d,rcdirend) | |
|
1142 | try: | |
|
1143 | rcdir = filter(os.path.isdir,map(cfg,sys.path))[0] | |
|
1144 | print "Initializing from configuration",rcdir | |
|
1145 | except IndexError: | |
|
1146 | warning = """ | |
|
1147 | Installation error. IPython's directory was not found. | |
|
1148 | ||
|
1149 | Check the following: | |
|
1150 | ||
|
1151 | The ipython/IPython directory should be in a directory belonging to your | |
|
1152 | PYTHONPATH environment variable (that is, it should be in a directory | |
|
1153 | belonging to sys.path). You can copy it explicitly there or just link to it. | |
|
1154 | ||
|
1155 | IPython will create a minimal default configuration for you. | |
|
1156 | ||
|
1157 | """ | |
|
1158 | warn(warning) | |
|
1159 | wait() | |
|
1160 | ||
|
1161 | if sys.platform =='win32': | |
|
1162 | inif = 'ipythonrc.ini' | |
|
1163 | else: | |
|
1164 | inif = 'ipythonrc' | |
|
1165 | minimal_setup = {'ipy_user_conf.py' : 'import ipy_defaults', | |
|
1166 | inif : '# intentionally left blank' } | |
|
1167 | os.makedirs(ipythondir, mode = 0777) | |
|
1168 | for f, cont in minimal_setup.items(): | |
|
1169 | open(ipythondir + '/' + f,'w').write(cont) | |
|
1170 | ||
|
1171 | return | |
|
1172 | ||
|
1173 | if mode == 'install': | |
|
1174 | try: | |
|
1175 | shutil.copytree(rcdir,ipythondir) | |
|
1176 | os.chdir(ipythondir) | |
|
1177 | rc_files = glb("ipythonrc*") | |
|
1178 | for rc_file in rc_files: | |
|
1179 | os.rename(rc_file,rc_file+rc_suffix) | |
|
1180 | except: | |
|
1181 | warning = """ | |
|
1182 | ||
|
1183 | There was a problem with the installation: | |
|
1184 | %s | |
|
1185 | Try to correct it or contact the developers if you think it's a bug. | |
|
1186 | IPython will proceed with builtin defaults.""" % sys.exc_info()[1] | |
|
1187 | warn(warning) | |
|
1188 | wait() | |
|
1189 | return | |
|
1190 | ||
|
1191 | elif mode == 'upgrade': | |
|
1192 | try: | |
|
1193 | os.chdir(ipythondir) | |
|
1194 | except: | |
|
1195 | print """ | |
|
1196 | Can not upgrade: changing to directory %s failed. Details: | |
|
1197 | %s | |
|
1198 | """ % (ipythondir,sys.exc_info()[1]) | |
|
1199 | wait() | |
|
1200 | return | |
|
1201 | else: | |
|
1202 | sources = glb(os.path.join(rcdir,'[A-Za-z]*')) | |
|
1203 | for new_full_path in sources: | |
|
1204 | new_filename = os.path.basename(new_full_path) | |
|
1205 | if new_filename.startswith('ipythonrc'): | |
|
1206 | new_filename = new_filename + rc_suffix | |
|
1207 | # The config directory should only contain files, skip any | |
|
1208 | # directories which may be there (like CVS) | |
|
1209 | if os.path.isdir(new_full_path): | |
|
1210 | continue | |
|
1211 | if os.path.exists(new_filename): | |
|
1212 | old_file = new_filename+'.old' | |
|
1213 | if os.path.exists(old_file): | |
|
1214 | os.remove(old_file) | |
|
1215 | os.rename(new_filename,old_file) | |
|
1216 | shutil.copy(new_full_path,new_filename) | |
|
1217 | else: | |
|
1218 | raise ValueError,'unrecognized mode for install:',`mode` | |
|
1219 | ||
|
1220 | # Fix line-endings to those native to each platform in the config | |
|
1221 | # directory. | |
|
1222 | try: | |
|
1223 | os.chdir(ipythondir) | |
|
1224 | except: | |
|
1225 | print """ | |
|
1226 | Problem: changing to directory %s failed. | |
|
1227 | Details: | |
|
1228 | %s | |
|
1229 | ||
|
1230 | Some configuration files may have incorrect line endings. This should not | |
|
1231 | cause any problems during execution. """ % (ipythondir,sys.exc_info()[1]) | |
|
1232 | wait() | |
|
1233 | else: | |
|
1234 | for fname in glb('ipythonrc*'): | |
|
1235 | try: | |
|
1236 | native_line_ends(fname,backup=0) | |
|
1237 | except IOError: | |
|
1238 | pass | |
|
1239 | ||
|
1240 | if mode == 'install': | |
|
1241 | print """ | |
|
1242 | Successful installation! | |
|
1243 | ||
|
1244 | Please read the sections 'Initial Configuration' and 'Quick Tips' in the | |
|
1245 | IPython manual (there are both HTML and PDF versions supplied with the | |
|
1246 | distribution) to make sure that your system environment is properly configured | |
|
1247 | to take advantage of IPython's features. | |
|
1248 | ||
|
1249 | Important note: the configuration system has changed! The old system is | |
|
1250 | still in place, but its setting may be partly overridden by the settings in | |
|
1251 | "~/.ipython/ipy_user_conf.py" config file. Please take a look at the file | |
|
1252 | if some of the new settings bother you. | |
|
1253 | ||
|
1254 | """ | |
|
1255 | else: | |
|
1256 | print """ | |
|
1257 | Successful upgrade! | |
|
1258 | ||
|
1259 | All files in your directory: | |
|
1260 | %(ipythondir)s | |
|
1261 | which would have been overwritten by the upgrade were backed up with a .old | |
|
1262 | extension. If you had made particular customizations in those files you may | |
|
1263 | want to merge them back into the new files.""" % locals() | |
|
1264 | wait() | |
|
1265 | os.chdir(cwd) | |
|
1266 | # end user_setup() | |
|
1319 | Note | |
|
1320 | ---- | |
|
1321 | DEPRECATED: use the top-level user_setup() function instead. | |
|
1322 | """ | |
|
1323 | return user_setup(ipythondir,rc_suffix,mode) | |
|
1267 | 1324 | |
|
1268 | 1325 | def atexit_operations(self): |
|
1269 | 1326 | """This will be executed at the time of exit. |
|
1270 | 1327 | |
|
1271 | 1328 | Saving of persistent data should be performed here. """ |
|
1272 | 1329 | |
|
1273 | 1330 | #print '*** IPython exit cleanup ***' # dbg |
|
1274 | 1331 | # input history |
|
1275 | 1332 | self.savehist() |
|
1276 | 1333 | |
|
1277 | 1334 | # Cleanup all tempfiles left around |
|
1278 | 1335 | for tfile in self.tempfiles: |
|
1279 | 1336 | try: |
|
1280 | 1337 | os.unlink(tfile) |
|
1281 | 1338 | except OSError: |
|
1282 | 1339 | pass |
|
1283 | 1340 | |
|
1284 | 1341 | # Clear all user namespaces to release all references cleanly. |
|
1285 | 1342 | self.reset() |
|
1286 | 1343 | |
|
1287 | 1344 | # Run user hooks |
|
1288 | 1345 | self.hooks.shutdown_hook() |
|
1289 | 1346 | |
|
1290 | 1347 | def reset(self): |
|
1291 | 1348 | """Clear all internal namespaces. |
|
1292 | 1349 | |
|
1293 | 1350 | Note that this is much more aggressive than %reset, since it clears |
|
1294 | 1351 | fully all namespaces, as well as all input/output lists. |
|
1295 | 1352 | """ |
|
1296 | 1353 | for ns in self.ns_refs_table: |
|
1297 | 1354 | ns.clear() |
|
1298 | 1355 | |
|
1299 | 1356 | # Clear input and output histories |
|
1300 | 1357 | self.input_hist[:] = [] |
|
1301 | 1358 | self.input_hist_raw[:] = [] |
|
1302 | 1359 | self.output_hist.clear() |
|
1303 | 1360 | # Restore the user namespaces to minimal usability |
|
1304 | 1361 | self.init_namespaces() |
|
1305 | 1362 | |
|
1306 | 1363 | def savehist(self): |
|
1307 | 1364 | """Save input history to a file (via readline library).""" |
|
1308 | 1365 | |
|
1309 | 1366 | if not self.has_readline: |
|
1310 | 1367 | return |
|
1311 | 1368 | |
|
1312 | 1369 | try: |
|
1313 | 1370 | self.readline.write_history_file(self.histfile) |
|
1314 | 1371 | except: |
|
1315 | 1372 | print 'Unable to save IPython command history to file: ' + \ |
|
1316 | 1373 | `self.histfile` |
|
1317 | 1374 | |
|
1318 | 1375 | def reloadhist(self): |
|
1319 | 1376 | """Reload the input history from disk file.""" |
|
1320 | 1377 | |
|
1321 | 1378 | if self.has_readline: |
|
1322 | 1379 | try: |
|
1323 | 1380 | self.readline.clear_history() |
|
1324 | 1381 | self.readline.read_history_file(self.shell.histfile) |
|
1325 | 1382 | except AttributeError: |
|
1326 | 1383 | pass |
|
1327 | 1384 | |
|
1328 | 1385 | |
|
1329 | 1386 | def history_saving_wrapper(self, func): |
|
1330 | 1387 | """ Wrap func for readline history saving |
|
1331 | 1388 | |
|
1332 | 1389 | Convert func into callable that saves & restores |
|
1333 | 1390 | history around the call """ |
|
1334 | 1391 | |
|
1335 | 1392 | if not self.has_readline: |
|
1336 | 1393 | return func |
|
1337 | 1394 | |
|
1338 | 1395 | def wrapper(): |
|
1339 | 1396 | self.savehist() |
|
1340 | 1397 | try: |
|
1341 | 1398 | func() |
|
1342 | 1399 | finally: |
|
1343 | 1400 | readline.read_history_file(self.histfile) |
|
1344 | 1401 | return wrapper |
|
1345 | 1402 | |
|
1346 | 1403 | def pre_readline(self): |
|
1347 | 1404 | """readline hook to be used at the start of each line. |
|
1348 | 1405 | |
|
1349 | 1406 | Currently it handles auto-indent only.""" |
|
1350 | 1407 | |
|
1351 | 1408 | #debugx('self.indent_current_nsp','pre_readline:') |
|
1352 | 1409 | |
|
1353 | 1410 | if self.rl_do_indent: |
|
1354 | 1411 | self.readline.insert_text(self.indent_current_str()) |
|
1355 | 1412 | if self.rl_next_input is not None: |
|
1356 | 1413 | self.readline.insert_text(self.rl_next_input) |
|
1357 | 1414 | self.rl_next_input = None |
|
1358 | 1415 | |
|
1359 | 1416 | def init_readline(self): |
|
1360 | 1417 | """Command history completion/saving/reloading.""" |
|
1361 | 1418 | |
|
1362 | 1419 | |
|
1363 | 1420 | import IPython.rlineimpl as readline |
|
1364 | 1421 | |
|
1365 | 1422 | if not readline.have_readline: |
|
1366 | 1423 | self.has_readline = 0 |
|
1367 | 1424 | self.readline = None |
|
1368 | 1425 | # no point in bugging windows users with this every time: |
|
1369 | 1426 | warn('Readline services not available on this platform.') |
|
1370 | 1427 | else: |
|
1371 | 1428 | sys.modules['readline'] = readline |
|
1372 | 1429 | import atexit |
|
1373 | 1430 | from IPython.completer import IPCompleter |
|
1374 | 1431 | self.Completer = IPCompleter(self, |
|
1375 | 1432 | self.user_ns, |
|
1376 | 1433 | self.user_global_ns, |
|
1377 | 1434 | self.rc.readline_omit__names, |
|
1378 | 1435 | self.alias_table) |
|
1379 | 1436 | sdisp = self.strdispatchers.get('complete_command', StrDispatch()) |
|
1380 | 1437 | self.strdispatchers['complete_command'] = sdisp |
|
1381 | 1438 | self.Completer.custom_completers = sdisp |
|
1382 | 1439 | # Platform-specific configuration |
|
1383 | 1440 | if os.name == 'nt': |
|
1384 | 1441 | self.readline_startup_hook = readline.set_pre_input_hook |
|
1385 | 1442 | else: |
|
1386 | 1443 | self.readline_startup_hook = readline.set_startup_hook |
|
1387 | 1444 | |
|
1388 | 1445 | # Load user's initrc file (readline config) |
|
1389 | 1446 | # Or if libedit is used, load editrc. |
|
1390 | 1447 | inputrc_name = os.environ.get('INPUTRC') |
|
1391 | 1448 | if inputrc_name is None: |
|
1392 | 1449 | home_dir = get_home_dir() |
|
1393 | 1450 | if home_dir is not None: |
|
1394 | 1451 | inputrc_name = '.inputrc' |
|
1395 | 1452 | if readline.uses_libedit: |
|
1396 | 1453 | inputrc_name = '.editrc' |
|
1397 | 1454 | inputrc_name = os.path.join(home_dir, inputrc_name) |
|
1398 | 1455 | if os.path.isfile(inputrc_name): |
|
1399 | 1456 | try: |
|
1400 | 1457 | readline.read_init_file(inputrc_name) |
|
1401 | 1458 | except: |
|
1402 | 1459 | warn('Problems reading readline initialization file <%s>' |
|
1403 | 1460 | % inputrc_name) |
|
1404 | 1461 | |
|
1405 | 1462 | self.has_readline = 1 |
|
1406 | 1463 | self.readline = readline |
|
1407 | 1464 | # save this in sys so embedded copies can restore it properly |
|
1408 | 1465 | sys.ipcompleter = self.Completer.complete |
|
1409 | 1466 | self.set_completer() |
|
1410 | 1467 | |
|
1411 | 1468 | # Configure readline according to user's prefs |
|
1412 | 1469 | # This is only done if GNU readline is being used. If libedit |
|
1413 | 1470 | # is being used (as on Leopard) the readline config is |
|
1414 | 1471 | # not run as the syntax for libedit is different. |
|
1415 | 1472 | if not readline.uses_libedit: |
|
1416 | 1473 | for rlcommand in self.rc.readline_parse_and_bind: |
|
1417 | 1474 | #print "loading rl:",rlcommand # dbg |
|
1418 | 1475 | readline.parse_and_bind(rlcommand) |
|
1419 | 1476 | |
|
1420 | 1477 | # remove some chars from the delimiters list |
|
1421 | 1478 | delims = readline.get_completer_delims() |
|
1422 | 1479 | delims = delims.translate(string._idmap, |
|
1423 | 1480 | self.rc.readline_remove_delims) |
|
1424 | 1481 | readline.set_completer_delims(delims) |
|
1425 | 1482 | # otherwise we end up with a monster history after a while: |
|
1426 | 1483 | readline.set_history_length(1000) |
|
1427 | 1484 | try: |
|
1428 | 1485 | #print '*** Reading readline history' # dbg |
|
1429 | 1486 | readline.read_history_file(self.histfile) |
|
1430 | 1487 | except IOError: |
|
1431 | 1488 | pass # It doesn't exist yet. |
|
1432 | 1489 | |
|
1433 | 1490 | atexit.register(self.atexit_operations) |
|
1434 | 1491 | del atexit |
|
1435 | 1492 | |
|
1436 | 1493 | # Configure auto-indent for all platforms |
|
1437 | 1494 | self.set_autoindent(self.rc.autoindent) |
|
1438 | 1495 | |
|
1439 | 1496 | def ask_yes_no(self,prompt,default=True): |
|
1440 | 1497 | if self.rc.quiet: |
|
1441 | 1498 | return True |
|
1442 | 1499 | return ask_yes_no(prompt,default) |
|
1443 | 1500 | |
|
1444 |
def |
|
|
1445 | """Cache a main module. | |
|
1501 | def new_main_mod(self,ns=None): | |
|
1502 | """Return a new 'main' module object for user code execution. | |
|
1503 | """ | |
|
1504 | main_mod = self._user_main_module | |
|
1505 | init_fakemod_dict(main_mod,ns) | |
|
1506 | return main_mod | |
|
1507 | ||
|
1508 | def cache_main_mod(self,ns,fname): | |
|
1509 | """Cache a main module's namespace. | |
|
1446 | 1510 | |
|
1447 |
When scripts are executed via %run, we must keep a reference to the |
|
|
1448 |
__main__ module (a FakeModule instance) around so |
|
|
1449 |
clear it, rendering objects defined therein |
|
|
1511 | When scripts are executed via %run, we must keep a reference to the | |
|
1512 | namespace of their __main__ module (a FakeModule instance) around so | |
|
1513 | that Python doesn't clear it, rendering objects defined therein | |
|
1514 | useless. | |
|
1450 | 1515 | |
|
1451 | 1516 | This method keeps said reference in a private dict, keyed by the |
|
1452 | 1517 | absolute path of the module object (which corresponds to the script |
|
1453 | 1518 | path). This way, for multiple executions of the same script we only |
|
1454 |
keep one copy of |
|
|
1455 |
from old references while allowing the objects from the last |
|
|
1456 | to be accessible. | |
|
1519 | keep one copy of the namespace (the last one), thus preventing memory | |
|
1520 | leaks from old references while allowing the objects from the last | |
|
1521 | execution to be accessible. | |
|
1522 | ||
|
1523 | Note: we can not allow the actual FakeModule instances to be deleted, | |
|
1524 | because of how Python tears down modules (it hard-sets all their | |
|
1525 | references to None without regard for reference counts). This method | |
|
1526 | must therefore make a *copy* of the given namespace, to allow the | |
|
1527 | original module's __dict__ to be cleared and reused. | |
|
1457 | 1528 | |
|
1529 | ||
|
1458 | 1530 | Parameters |
|
1459 | 1531 | ---------- |
|
1460 | mod : a module object | |
|
1532 | ns : a namespace (a dict, typically) | |
|
1533 | ||
|
1534 | fname : str | |
|
1535 | Filename associated with the namespace. | |
|
1461 | 1536 | |
|
1462 | 1537 | Examples |
|
1463 | 1538 | -------- |
|
1464 | 1539 | |
|
1465 | 1540 | In [10]: import IPython |
|
1466 | 1541 | |
|
1467 | In [11]: _ip.IP.cache_main_mod(IPython) | |
|
1542 | In [11]: _ip.IP.cache_main_mod(IPython.__dict__,IPython.__file__) | |
|
1468 | 1543 | |
|
1469 |
In [12]: IPython.__file__ in _ip.IP._ |
|
|
1544 | In [12]: IPython.__file__ in _ip.IP._main_ns_cache | |
|
1470 | 1545 | Out[12]: True |
|
1471 | 1546 | """ |
|
1472 |
self. |
|
|
1547 | self._main_ns_cache[os.path.abspath(fname)] = ns.copy() | |
|
1473 | 1548 | |
|
1474 | 1549 | def clear_main_mod_cache(self): |
|
1475 | 1550 | """Clear the cache of main modules. |
|
1476 | 1551 | |
|
1477 | 1552 | Mainly for use by utilities like %reset. |
|
1478 | 1553 | |
|
1479 | 1554 | Examples |
|
1480 | 1555 | -------- |
|
1481 | 1556 | |
|
1482 | 1557 | In [15]: import IPython |
|
1483 | 1558 | |
|
1484 | In [16]: _ip.IP.cache_main_mod(IPython) | |
|
1559 | In [16]: _ip.IP.cache_main_mod(IPython.__dict__,IPython.__file__) | |
|
1485 | 1560 | |
|
1486 |
In [17]: len(_ip.IP._ |
|
|
1561 | In [17]: len(_ip.IP._main_ns_cache) > 0 | |
|
1487 | 1562 | Out[17]: True |
|
1488 | 1563 | |
|
1489 | 1564 | In [18]: _ip.IP.clear_main_mod_cache() |
|
1490 | 1565 | |
|
1491 |
In [19]: len(_ip.IP._ |
|
|
1566 | In [19]: len(_ip.IP._main_ns_cache) == 0 | |
|
1492 | 1567 | Out[19]: True |
|
1493 | 1568 | """ |
|
1494 |
self. |
|
|
1569 | self._main_ns_cache.clear() | |
|
1495 | 1570 | |
|
1496 | 1571 | def _should_recompile(self,e): |
|
1497 | 1572 | """Utility routine for edit_syntax_error""" |
|
1498 | 1573 | |
|
1499 | 1574 | if e.filename in ('<ipython console>','<input>','<string>', |
|
1500 | 1575 | '<console>','<BackgroundJob compilation>', |
|
1501 | 1576 | None): |
|
1502 | 1577 | |
|
1503 | 1578 | return False |
|
1504 | 1579 | try: |
|
1505 | 1580 | if (self.rc.autoedit_syntax and |
|
1506 | 1581 | not self.ask_yes_no('Return to editor to correct syntax error? ' |
|
1507 | 1582 | '[Y/n] ','y')): |
|
1508 | 1583 | return False |
|
1509 | 1584 | except EOFError: |
|
1510 | 1585 | return False |
|
1511 | 1586 | |
|
1512 | 1587 | def int0(x): |
|
1513 | 1588 | try: |
|
1514 | 1589 | return int(x) |
|
1515 | 1590 | except TypeError: |
|
1516 | 1591 | return 0 |
|
1517 | 1592 | # always pass integer line and offset values to editor hook |
|
1518 | 1593 | try: |
|
1519 | 1594 | self.hooks.fix_error_editor(e.filename, |
|
1520 | 1595 | int0(e.lineno),int0(e.offset),e.msg) |
|
1521 | 1596 | except IPython.ipapi.TryNext: |
|
1522 | 1597 | warn('Could not open editor') |
|
1523 | 1598 | return False |
|
1524 | 1599 | return True |
|
1525 | 1600 | |
|
1526 | 1601 | def edit_syntax_error(self): |
|
1527 | 1602 | """The bottom half of the syntax error handler called in the main loop. |
|
1528 | 1603 | |
|
1529 | 1604 | Loop until syntax error is fixed or user cancels. |
|
1530 | 1605 | """ |
|
1531 | 1606 | |
|
1532 | 1607 | while self.SyntaxTB.last_syntax_error: |
|
1533 | 1608 | # copy and clear last_syntax_error |
|
1534 | 1609 | err = self.SyntaxTB.clear_err_state() |
|
1535 | 1610 | if not self._should_recompile(err): |
|
1536 | 1611 | return |
|
1537 | 1612 | try: |
|
1538 | 1613 | # may set last_syntax_error again if a SyntaxError is raised |
|
1539 | 1614 | self.safe_execfile(err.filename,self.user_ns) |
|
1540 | 1615 | except: |
|
1541 | 1616 | self.showtraceback() |
|
1542 | 1617 | else: |
|
1543 | 1618 | try: |
|
1544 | 1619 | f = file(err.filename) |
|
1545 | 1620 | try: |
|
1546 | 1621 | sys.displayhook(f.read()) |
|
1547 | 1622 | finally: |
|
1548 | 1623 | f.close() |
|
1549 | 1624 | except: |
|
1550 | 1625 | self.showtraceback() |
|
1551 | 1626 | |
|
1552 | 1627 | def showsyntaxerror(self, filename=None): |
|
1553 | 1628 | """Display the syntax error that just occurred. |
|
1554 | 1629 | |
|
1555 | 1630 | This doesn't display a stack trace because there isn't one. |
|
1556 | 1631 | |
|
1557 | 1632 | If a filename is given, it is stuffed in the exception instead |
|
1558 | 1633 | of what was there before (because Python's parser always uses |
|
1559 | 1634 | "<string>" when reading from a string). |
|
1560 | 1635 | """ |
|
1561 | 1636 | etype, value, last_traceback = sys.exc_info() |
|
1562 | 1637 | |
|
1563 | 1638 | # See note about these variables in showtraceback() below |
|
1564 | 1639 | sys.last_type = etype |
|
1565 | 1640 | sys.last_value = value |
|
1566 | 1641 | sys.last_traceback = last_traceback |
|
1567 | 1642 | |
|
1568 | 1643 | if filename and etype is SyntaxError: |
|
1569 | 1644 | # Work hard to stuff the correct filename in the exception |
|
1570 | 1645 | try: |
|
1571 | 1646 | msg, (dummy_filename, lineno, offset, line) = value |
|
1572 | 1647 | except: |
|
1573 | 1648 | # Not the format we expect; leave it alone |
|
1574 | 1649 | pass |
|
1575 | 1650 | else: |
|
1576 | 1651 | # Stuff in the right filename |
|
1577 | 1652 | try: |
|
1578 | 1653 | # Assume SyntaxError is a class exception |
|
1579 | 1654 | value = SyntaxError(msg, (filename, lineno, offset, line)) |
|
1580 | 1655 | except: |
|
1581 | 1656 | # If that failed, assume SyntaxError is a string |
|
1582 | 1657 | value = msg, (filename, lineno, offset, line) |
|
1583 | 1658 | self.SyntaxTB(etype,value,[]) |
|
1584 | 1659 | |
|
1585 | 1660 | def debugger(self,force=False): |
|
1586 | 1661 | """Call the pydb/pdb debugger. |
|
1587 | 1662 | |
|
1588 | 1663 | Keywords: |
|
1589 | 1664 | |
|
1590 | 1665 | - force(False): by default, this routine checks the instance call_pdb |
|
1591 | 1666 | flag and does not actually invoke the debugger if the flag is false. |
|
1592 | 1667 | The 'force' option forces the debugger to activate even if the flag |
|
1593 | 1668 | is false. |
|
1594 | 1669 | """ |
|
1595 | 1670 | |
|
1596 | 1671 | if not (force or self.call_pdb): |
|
1597 | 1672 | return |
|
1598 | 1673 | |
|
1599 | 1674 | if not hasattr(sys,'last_traceback'): |
|
1600 | 1675 | error('No traceback has been produced, nothing to debug.') |
|
1601 | 1676 | return |
|
1602 | 1677 | |
|
1603 | 1678 | # use pydb if available |
|
1604 | 1679 | if Debugger.has_pydb: |
|
1605 | 1680 | from pydb import pm |
|
1606 | 1681 | else: |
|
1607 | 1682 | # fallback to our internal debugger |
|
1608 | 1683 | pm = lambda : self.InteractiveTB.debugger(force=True) |
|
1609 | 1684 | self.history_saving_wrapper(pm)() |
|
1610 | 1685 | |
|
1611 | 1686 | def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None): |
|
1612 | 1687 | """Display the exception that just occurred. |
|
1613 | 1688 | |
|
1614 | 1689 | If nothing is known about the exception, this is the method which |
|
1615 | 1690 | should be used throughout the code for presenting user tracebacks, |
|
1616 | 1691 | rather than directly invoking the InteractiveTB object. |
|
1617 | 1692 | |
|
1618 | 1693 | A specific showsyntaxerror() also exists, but this method can take |
|
1619 | 1694 | care of calling it if needed, so unless you are explicitly catching a |
|
1620 | 1695 | SyntaxError exception, don't try to analyze the stack manually and |
|
1621 | 1696 | simply call this method.""" |
|
1622 | 1697 | |
|
1623 | 1698 | |
|
1624 | 1699 | # Though this won't be called by syntax errors in the input line, |
|
1625 | 1700 | # there may be SyntaxError cases whith imported code. |
|
1626 | 1701 | |
|
1627 | 1702 | try: |
|
1628 | 1703 | if exc_tuple is None: |
|
1629 | 1704 | etype, value, tb = sys.exc_info() |
|
1630 | 1705 | else: |
|
1631 | 1706 | etype, value, tb = exc_tuple |
|
1632 | 1707 | |
|
1633 | 1708 | if etype is SyntaxError: |
|
1634 | 1709 | self.showsyntaxerror(filename) |
|
1635 | 1710 | elif etype is IPython.ipapi.UsageError: |
|
1636 | 1711 | print "UsageError:", value |
|
1637 | 1712 | else: |
|
1638 | 1713 | # WARNING: these variables are somewhat deprecated and not |
|
1639 | 1714 | # necessarily safe to use in a threaded environment, but tools |
|
1640 | 1715 | # like pdb depend on their existence, so let's set them. If we |
|
1641 | 1716 | # find problems in the field, we'll need to revisit their use. |
|
1642 | 1717 | sys.last_type = etype |
|
1643 | 1718 | sys.last_value = value |
|
1644 | 1719 | sys.last_traceback = tb |
|
1645 | 1720 | |
|
1646 | 1721 | if etype in self.custom_exceptions: |
|
1647 | 1722 | self.CustomTB(etype,value,tb) |
|
1648 | 1723 | else: |
|
1649 | 1724 | self.InteractiveTB(etype,value,tb,tb_offset=tb_offset) |
|
1650 | 1725 | if self.InteractiveTB.call_pdb and self.has_readline: |
|
1651 | 1726 | # pdb mucks up readline, fix it back |
|
1652 | 1727 | self.set_completer() |
|
1653 | 1728 | except KeyboardInterrupt: |
|
1654 | 1729 | self.write("\nKeyboardInterrupt\n") |
|
1655 | 1730 | |
|
1656 | 1731 | def mainloop(self,banner=None): |
|
1657 | 1732 | """Creates the local namespace and starts the mainloop. |
|
1658 | 1733 | |
|
1659 | 1734 | If an optional banner argument is given, it will override the |
|
1660 | 1735 | internally created default banner.""" |
|
1661 | 1736 | |
|
1662 | 1737 | if self.rc.c: # Emulate Python's -c option |
|
1663 | 1738 | self.exec_init_cmd() |
|
1664 | 1739 | if banner is None: |
|
1665 | 1740 | if not self.rc.banner: |
|
1666 | 1741 | banner = '' |
|
1667 | 1742 | # banner is string? Use it directly! |
|
1668 | 1743 | elif isinstance(self.rc.banner,basestring): |
|
1669 | 1744 | banner = self.rc.banner |
|
1670 | 1745 | else: |
|
1671 | 1746 | banner = self.BANNER+self.banner2 |
|
1672 | 1747 | |
|
1673 | 1748 | # if you run stuff with -c <cmd>, raw hist is not updated |
|
1674 | 1749 | # ensure that it's in sync |
|
1675 | 1750 | if len(self.input_hist) != len (self.input_hist_raw): |
|
1676 | 1751 | self.input_hist_raw = InputList(self.input_hist) |
|
1677 | 1752 | |
|
1678 | 1753 | while 1: |
|
1679 | 1754 | try: |
|
1680 | 1755 | self.interact(banner) |
|
1681 | 1756 | #self.interact_with_readline() |
|
1682 | 1757 | |
|
1683 | 1758 | # XXX for testing of a readline-decoupled repl loop, call |
|
1684 | 1759 | # interact_with_readline above |
|
1685 | 1760 | |
|
1686 | 1761 | break |
|
1687 | 1762 | except KeyboardInterrupt: |
|
1688 | 1763 | # this should not be necessary, but KeyboardInterrupt |
|
1689 | 1764 | # handling seems rather unpredictable... |
|
1690 | 1765 | self.write("\nKeyboardInterrupt in interact()\n") |
|
1691 | 1766 | |
|
1692 | 1767 | def exec_init_cmd(self): |
|
1693 | 1768 | """Execute a command given at the command line. |
|
1694 | 1769 | |
|
1695 | 1770 | This emulates Python's -c option.""" |
|
1696 | 1771 | |
|
1697 | 1772 | #sys.argv = ['-c'] |
|
1698 | 1773 | self.push(self.prefilter(self.rc.c, False)) |
|
1699 | 1774 | if not self.rc.interact: |
|
1700 | 1775 | self.ask_exit() |
|
1701 | 1776 | |
|
1702 | 1777 | def embed_mainloop(self,header='',local_ns=None,global_ns=None,stack_depth=0): |
|
1703 | 1778 | """Embeds IPython into a running python program. |
|
1704 | 1779 | |
|
1705 | 1780 | Input: |
|
1706 | 1781 | |
|
1707 | 1782 | - header: An optional header message can be specified. |
|
1708 | 1783 | |
|
1709 | 1784 | - local_ns, global_ns: working namespaces. If given as None, the |
|
1710 | 1785 | IPython-initialized one is updated with __main__.__dict__, so that |
|
1711 | 1786 | program variables become visible but user-specific configuration |
|
1712 | 1787 | remains possible. |
|
1713 | 1788 | |
|
1714 | 1789 | - stack_depth: specifies how many levels in the stack to go to |
|
1715 | 1790 | looking for namespaces (when local_ns and global_ns are None). This |
|
1716 | 1791 | allows an intermediate caller to make sure that this function gets |
|
1717 | 1792 | the namespace from the intended level in the stack. By default (0) |
|
1718 | 1793 | it will get its locals and globals from the immediate caller. |
|
1719 | 1794 | |
|
1720 | 1795 | Warning: it's possible to use this in a program which is being run by |
|
1721 | 1796 | IPython itself (via %run), but some funny things will happen (a few |
|
1722 | 1797 | globals get overwritten). In the future this will be cleaned up, as |
|
1723 | 1798 | there is no fundamental reason why it can't work perfectly.""" |
|
1724 | 1799 | |
|
1725 | 1800 | # Get locals and globals from caller |
|
1726 | 1801 | if local_ns is None or global_ns is None: |
|
1727 | 1802 | call_frame = sys._getframe(stack_depth).f_back |
|
1728 | 1803 | |
|
1729 | 1804 | if local_ns is None: |
|
1730 | 1805 | local_ns = call_frame.f_locals |
|
1731 | 1806 | if global_ns is None: |
|
1732 | 1807 | global_ns = call_frame.f_globals |
|
1733 | 1808 | |
|
1734 | 1809 | # Update namespaces and fire up interpreter |
|
1735 | 1810 | |
|
1736 | 1811 | # The global one is easy, we can just throw it in |
|
1737 | 1812 | self.user_global_ns = global_ns |
|
1738 | 1813 | |
|
1739 | 1814 | # but the user/local one is tricky: ipython needs it to store internal |
|
1740 | 1815 | # data, but we also need the locals. We'll copy locals in the user |
|
1741 | 1816 | # one, but will track what got copied so we can delete them at exit. |
|
1742 | 1817 | # This is so that a later embedded call doesn't see locals from a |
|
1743 | 1818 | # previous call (which most likely existed in a separate scope). |
|
1744 | 1819 | local_varnames = local_ns.keys() |
|
1745 | 1820 | self.user_ns.update(local_ns) |
|
1746 | 1821 | #self.user_ns['local_ns'] = local_ns # dbg |
|
1747 | 1822 | |
|
1748 | 1823 | # Patch for global embedding to make sure that things don't overwrite |
|
1749 | 1824 | # user globals accidentally. Thanks to Richard <rxe@renre-europe.com> |
|
1750 | 1825 | # FIXME. Test this a bit more carefully (the if.. is new) |
|
1751 | 1826 | if local_ns is None and global_ns is None: |
|
1752 | 1827 | self.user_global_ns.update(__main__.__dict__) |
|
1753 | 1828 | |
|
1754 | 1829 | # make sure the tab-completer has the correct frame information, so it |
|
1755 | 1830 | # actually completes using the frame's locals/globals |
|
1756 | 1831 | self.set_completer_frame() |
|
1757 | 1832 | |
|
1758 | 1833 | # before activating the interactive mode, we need to make sure that |
|
1759 | 1834 | # all names in the builtin namespace needed by ipython point to |
|
1760 | 1835 | # ourselves, and not to other instances. |
|
1761 | 1836 | self.add_builtins() |
|
1762 | 1837 | |
|
1763 | 1838 | self.interact(header) |
|
1764 | 1839 | |
|
1765 | 1840 | # now, purge out the user namespace from anything we might have added |
|
1766 | 1841 | # from the caller's local namespace |
|
1767 | 1842 | delvar = self.user_ns.pop |
|
1768 | 1843 | for var in local_varnames: |
|
1769 | 1844 | delvar(var,None) |
|
1770 | 1845 | # and clean builtins we may have overridden |
|
1771 | 1846 | self.clean_builtins() |
|
1772 | 1847 | |
|
1773 | 1848 | def interact_prompt(self): |
|
1774 | 1849 | """ Print the prompt (in read-eval-print loop) |
|
1775 | 1850 | |
|
1776 | 1851 | Provided for those who want to implement their own read-eval-print loop (e.g. GUIs), not |
|
1777 | 1852 | used in standard IPython flow. |
|
1778 | 1853 | """ |
|
1779 | 1854 | if self.more: |
|
1780 | 1855 | try: |
|
1781 | 1856 | prompt = self.hooks.generate_prompt(True) |
|
1782 | 1857 | except: |
|
1783 | 1858 | self.showtraceback() |
|
1784 | 1859 | if self.autoindent: |
|
1785 | 1860 | self.rl_do_indent = True |
|
1786 | 1861 | |
|
1787 | 1862 | else: |
|
1788 | 1863 | try: |
|
1789 | 1864 | prompt = self.hooks.generate_prompt(False) |
|
1790 | 1865 | except: |
|
1791 | 1866 | self.showtraceback() |
|
1792 | 1867 | self.write(prompt) |
|
1793 | 1868 | |
|
1794 | 1869 | def interact_handle_input(self,line): |
|
1795 | 1870 | """ Handle the input line (in read-eval-print loop) |
|
1796 | 1871 | |
|
1797 | 1872 | Provided for those who want to implement their own read-eval-print loop (e.g. GUIs), not |
|
1798 | 1873 | used in standard IPython flow. |
|
1799 | 1874 | """ |
|
1800 | 1875 | if line.lstrip() == line: |
|
1801 | 1876 | self.shadowhist.add(line.strip()) |
|
1802 | 1877 | lineout = self.prefilter(line,self.more) |
|
1803 | 1878 | |
|
1804 | 1879 | if line.strip(): |
|
1805 | 1880 | if self.more: |
|
1806 | 1881 | self.input_hist_raw[-1] += '%s\n' % line |
|
1807 | 1882 | else: |
|
1808 | 1883 | self.input_hist_raw.append('%s\n' % line) |
|
1809 | 1884 | |
|
1810 | 1885 | |
|
1811 | 1886 | self.more = self.push(lineout) |
|
1812 | 1887 | if (self.SyntaxTB.last_syntax_error and |
|
1813 | 1888 | self.rc.autoedit_syntax): |
|
1814 | 1889 | self.edit_syntax_error() |
|
1815 | 1890 | |
|
1816 | 1891 | def interact_with_readline(self): |
|
1817 | 1892 | """ Demo of using interact_handle_input, interact_prompt |
|
1818 | 1893 | |
|
1819 | 1894 | This is the main read-eval-print loop. If you need to implement your own (e.g. for GUI), |
|
1820 | 1895 | it should work like this. |
|
1821 | 1896 | """ |
|
1822 | 1897 | self.readline_startup_hook(self.pre_readline) |
|
1823 | 1898 | while not self.exit_now: |
|
1824 | 1899 | self.interact_prompt() |
|
1825 | 1900 | if self.more: |
|
1826 | 1901 | self.rl_do_indent = True |
|
1827 | 1902 | else: |
|
1828 | 1903 | self.rl_do_indent = False |
|
1829 | 1904 | line = raw_input_original().decode(self.stdin_encoding) |
|
1830 | 1905 | self.interact_handle_input(line) |
|
1831 | 1906 | |
|
1832 | 1907 | |
|
1833 | 1908 | def interact(self, banner=None): |
|
1834 | 1909 | """Closely emulate the interactive Python console. |
|
1835 | 1910 | |
|
1836 | 1911 | The optional banner argument specify the banner to print |
|
1837 | 1912 | before the first interaction; by default it prints a banner |
|
1838 | 1913 | similar to the one printed by the real Python interpreter, |
|
1839 | 1914 | followed by the current class name in parentheses (so as not |
|
1840 | 1915 | to confuse this with the real interpreter -- since it's so |
|
1841 | 1916 | close!). |
|
1842 | 1917 | |
|
1843 | 1918 | """ |
|
1844 | 1919 | |
|
1845 | 1920 | if self.exit_now: |
|
1846 | 1921 | # batch run -> do not interact |
|
1847 | 1922 | return |
|
1848 | 1923 | cprt = 'Type "copyright", "credits" or "license" for more information.' |
|
1849 | 1924 | if banner is None: |
|
1850 | 1925 | self.write("Python %s on %s\n%s\n(%s)\n" % |
|
1851 | 1926 | (sys.version, sys.platform, cprt, |
|
1852 | 1927 | self.__class__.__name__)) |
|
1853 | 1928 | else: |
|
1854 | 1929 | self.write(banner) |
|
1855 | 1930 | |
|
1856 | 1931 | more = 0 |
|
1857 | 1932 | |
|
1858 | 1933 | # Mark activity in the builtins |
|
1859 | 1934 | __builtin__.__dict__['__IPYTHON__active'] += 1 |
|
1860 | 1935 | |
|
1861 | 1936 | if self.has_readline: |
|
1862 | 1937 | self.readline_startup_hook(self.pre_readline) |
|
1863 | 1938 | # exit_now is set by a call to %Exit or %Quit, through the |
|
1864 | 1939 | # ask_exit callback. |
|
1865 | 1940 | |
|
1866 | 1941 | while not self.exit_now: |
|
1867 | 1942 | self.hooks.pre_prompt_hook() |
|
1868 | 1943 | if more: |
|
1869 | 1944 | try: |
|
1870 | 1945 | prompt = self.hooks.generate_prompt(True) |
|
1871 | 1946 | except: |
|
1872 | 1947 | self.showtraceback() |
|
1873 | 1948 | if self.autoindent: |
|
1874 | 1949 | self.rl_do_indent = True |
|
1875 | 1950 | |
|
1876 | 1951 | else: |
|
1877 | 1952 | try: |
|
1878 | 1953 | prompt = self.hooks.generate_prompt(False) |
|
1879 | 1954 | except: |
|
1880 | 1955 | self.showtraceback() |
|
1881 | 1956 | try: |
|
1882 | 1957 | line = self.raw_input(prompt,more) |
|
1883 | 1958 | if self.exit_now: |
|
1884 | 1959 | # quick exit on sys.std[in|out] close |
|
1885 | 1960 | break |
|
1886 | 1961 | if self.autoindent: |
|
1887 | 1962 | self.rl_do_indent = False |
|
1888 | 1963 | |
|
1889 | 1964 | except KeyboardInterrupt: |
|
1890 | 1965 | #double-guard against keyboardinterrupts during kbdint handling |
|
1891 | 1966 | try: |
|
1892 | 1967 | self.write('\nKeyboardInterrupt\n') |
|
1893 | 1968 | self.resetbuffer() |
|
1894 | 1969 | # keep cache in sync with the prompt counter: |
|
1895 | 1970 | self.outputcache.prompt_count -= 1 |
|
1896 | 1971 | |
|
1897 | 1972 | if self.autoindent: |
|
1898 | 1973 | self.indent_current_nsp = 0 |
|
1899 | 1974 | more = 0 |
|
1900 | 1975 | except KeyboardInterrupt: |
|
1901 | 1976 | pass |
|
1902 | 1977 | except EOFError: |
|
1903 | 1978 | if self.autoindent: |
|
1904 | 1979 | self.rl_do_indent = False |
|
1905 | 1980 | self.readline_startup_hook(None) |
|
1906 | 1981 | self.write('\n') |
|
1907 | 1982 | self.exit() |
|
1908 | 1983 | except bdb.BdbQuit: |
|
1909 | 1984 | warn('The Python debugger has exited with a BdbQuit exception.\n' |
|
1910 | 1985 | 'Because of how pdb handles the stack, it is impossible\n' |
|
1911 | 1986 | 'for IPython to properly format this particular exception.\n' |
|
1912 | 1987 | 'IPython will resume normal operation.') |
|
1913 | 1988 | except: |
|
1914 | 1989 | # exceptions here are VERY RARE, but they can be triggered |
|
1915 | 1990 | # asynchronously by signal handlers, for example. |
|
1916 | 1991 | self.showtraceback() |
|
1917 | 1992 | else: |
|
1918 | 1993 | more = self.push(line) |
|
1919 | 1994 | if (self.SyntaxTB.last_syntax_error and |
|
1920 | 1995 | self.rc.autoedit_syntax): |
|
1921 | 1996 | self.edit_syntax_error() |
|
1922 | 1997 | |
|
1923 | 1998 | # We are off again... |
|
1924 | 1999 | __builtin__.__dict__['__IPYTHON__active'] -= 1 |
|
1925 | 2000 | |
|
1926 | 2001 | def excepthook(self, etype, value, tb): |
|
1927 | 2002 | """One more defense for GUI apps that call sys.excepthook. |
|
1928 | 2003 | |
|
1929 | 2004 | GUI frameworks like wxPython trap exceptions and call |
|
1930 | 2005 | sys.excepthook themselves. I guess this is a feature that |
|
1931 | 2006 | enables them to keep running after exceptions that would |
|
1932 | 2007 | otherwise kill their mainloop. This is a bother for IPython |
|
1933 | 2008 | which excepts to catch all of the program exceptions with a try: |
|
1934 | 2009 | except: statement. |
|
1935 | 2010 | |
|
1936 | 2011 | Normally, IPython sets sys.excepthook to a CrashHandler instance, so if |
|
1937 | 2012 | any app directly invokes sys.excepthook, it will look to the user like |
|
1938 | 2013 | IPython crashed. In order to work around this, we can disable the |
|
1939 | 2014 | CrashHandler and replace it with this excepthook instead, which prints a |
|
1940 | 2015 | regular traceback using our InteractiveTB. In this fashion, apps which |
|
1941 | 2016 | call sys.excepthook will generate a regular-looking exception from |
|
1942 | 2017 | IPython, and the CrashHandler will only be triggered by real IPython |
|
1943 | 2018 | crashes. |
|
1944 | 2019 | |
|
1945 | 2020 | This hook should be used sparingly, only in places which are not likely |
|
1946 | 2021 | to be true IPython errors. |
|
1947 | 2022 | """ |
|
1948 | 2023 | self.showtraceback((etype,value,tb),tb_offset=0) |
|
1949 | 2024 | |
|
1950 | 2025 | def expand_aliases(self,fn,rest): |
|
1951 | 2026 | """ Expand multiple levels of aliases: |
|
1952 | 2027 | |
|
1953 | 2028 | if: |
|
1954 | 2029 | |
|
1955 | 2030 | alias foo bar /tmp |
|
1956 | 2031 | alias baz foo |
|
1957 | 2032 | |
|
1958 | 2033 | then: |
|
1959 | 2034 | |
|
1960 | 2035 | baz huhhahhei -> bar /tmp huhhahhei |
|
1961 | 2036 | |
|
1962 | 2037 | """ |
|
1963 | 2038 | line = fn + " " + rest |
|
1964 | 2039 | |
|
1965 | 2040 | done = set() |
|
1966 | 2041 | while 1: |
|
1967 | 2042 | pre,fn,rest = prefilter.splitUserInput(line, |
|
1968 | 2043 | prefilter.shell_line_split) |
|
1969 | 2044 | if fn in self.alias_table: |
|
1970 | 2045 | if fn in done: |
|
1971 | 2046 | warn("Cyclic alias definition, repeated '%s'" % fn) |
|
1972 | 2047 | return "" |
|
1973 | 2048 | done.add(fn) |
|
1974 | 2049 | |
|
1975 | 2050 | l2 = self.transform_alias(fn,rest) |
|
1976 | 2051 | # dir -> dir |
|
1977 | 2052 | # print "alias",line, "->",l2 #dbg |
|
1978 | 2053 | if l2 == line: |
|
1979 | 2054 | break |
|
1980 | 2055 | # ls -> ls -F should not recurse forever |
|
1981 | 2056 | if l2.split(None,1)[0] == line.split(None,1)[0]: |
|
1982 | 2057 | line = l2 |
|
1983 | 2058 | break |
|
1984 | 2059 | |
|
1985 | 2060 | line=l2 |
|
1986 | 2061 | |
|
1987 | 2062 | |
|
1988 | 2063 | # print "al expand to",line #dbg |
|
1989 | 2064 | else: |
|
1990 | 2065 | break |
|
1991 | 2066 | |
|
1992 | 2067 | return line |
|
1993 | 2068 | |
|
1994 | 2069 | def transform_alias(self, alias,rest=''): |
|
1995 | 2070 | """ Transform alias to system command string. |
|
1996 | 2071 | """ |
|
1997 | 2072 | trg = self.alias_table[alias] |
|
1998 | 2073 | |
|
1999 | 2074 | nargs,cmd = trg |
|
2000 | 2075 | # print trg #dbg |
|
2001 | 2076 | if ' ' in cmd and os.path.isfile(cmd): |
|
2002 | 2077 | cmd = '"%s"' % cmd |
|
2003 | 2078 | |
|
2004 | 2079 | # Expand the %l special to be the user's input line |
|
2005 | 2080 | if cmd.find('%l') >= 0: |
|
2006 | 2081 | cmd = cmd.replace('%l',rest) |
|
2007 | 2082 | rest = '' |
|
2008 | 2083 | if nargs==0: |
|
2009 | 2084 | # Simple, argument-less aliases |
|
2010 | 2085 | cmd = '%s %s' % (cmd,rest) |
|
2011 | 2086 | else: |
|
2012 | 2087 | # Handle aliases with positional arguments |
|
2013 | 2088 | args = rest.split(None,nargs) |
|
2014 | 2089 | if len(args)< nargs: |
|
2015 | 2090 | error('Alias <%s> requires %s arguments, %s given.' % |
|
2016 | 2091 | (alias,nargs,len(args))) |
|
2017 | 2092 | return None |
|
2018 | 2093 | cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:])) |
|
2019 | 2094 | # Now call the macro, evaluating in the user's namespace |
|
2020 | 2095 | #print 'new command: <%r>' % cmd # dbg |
|
2021 | 2096 | return cmd |
|
2022 | 2097 | |
|
2023 | 2098 | def call_alias(self,alias,rest=''): |
|
2024 | 2099 | """Call an alias given its name and the rest of the line. |
|
2025 | 2100 | |
|
2026 | 2101 | This is only used to provide backwards compatibility for users of |
|
2027 | 2102 | ipalias(), use of which is not recommended for anymore.""" |
|
2028 | 2103 | |
|
2029 | 2104 | # Now call the macro, evaluating in the user's namespace |
|
2030 | 2105 | cmd = self.transform_alias(alias, rest) |
|
2031 | 2106 | try: |
|
2032 | 2107 | self.system(cmd) |
|
2033 | 2108 | except: |
|
2034 | 2109 | self.showtraceback() |
|
2035 | 2110 | |
|
2036 | 2111 | def indent_current_str(self): |
|
2037 | 2112 | """return the current level of indentation as a string""" |
|
2038 | 2113 | return self.indent_current_nsp * ' ' |
|
2039 | 2114 | |
|
2040 | 2115 | def autoindent_update(self,line): |
|
2041 | 2116 | """Keep track of the indent level.""" |
|
2042 | 2117 | |
|
2043 | 2118 | #debugx('line') |
|
2044 | 2119 | #debugx('self.indent_current_nsp') |
|
2045 | 2120 | if self.autoindent: |
|
2046 | 2121 | if line: |
|
2047 | 2122 | inisp = num_ini_spaces(line) |
|
2048 | 2123 | if inisp < self.indent_current_nsp: |
|
2049 | 2124 | self.indent_current_nsp = inisp |
|
2050 | 2125 | |
|
2051 | 2126 | if line[-1] == ':': |
|
2052 | 2127 | self.indent_current_nsp += 4 |
|
2053 | 2128 | elif dedent_re.match(line): |
|
2054 | 2129 | self.indent_current_nsp -= 4 |
|
2055 | 2130 | else: |
|
2056 | 2131 | self.indent_current_nsp = 0 |
|
2057 | 2132 | |
|
2058 | 2133 | def runlines(self,lines): |
|
2059 | 2134 | """Run a string of one or more lines of source. |
|
2060 | 2135 | |
|
2061 | 2136 | This method is capable of running a string containing multiple source |
|
2062 | 2137 | lines, as if they had been entered at the IPython prompt. Since it |
|
2063 | 2138 | exposes IPython's processing machinery, the given strings can contain |
|
2064 | 2139 | magic calls (%magic), special shell access (!cmd), etc.""" |
|
2065 | 2140 | |
|
2066 | 2141 | # We must start with a clean buffer, in case this is run from an |
|
2067 | 2142 | # interactive IPython session (via a magic, for example). |
|
2068 | 2143 | self.resetbuffer() |
|
2069 | 2144 | lines = lines.split('\n') |
|
2070 | 2145 | more = 0 |
|
2071 | 2146 | |
|
2072 | 2147 | for line in lines: |
|
2073 | 2148 | # skip blank lines so we don't mess up the prompt counter, but do |
|
2074 | 2149 | # NOT skip even a blank line if we are in a code block (more is |
|
2075 | 2150 | # true) |
|
2076 | 2151 | |
|
2077 | 2152 | if line or more: |
|
2078 | 2153 | # push to raw history, so hist line numbers stay in sync |
|
2079 | 2154 | self.input_hist_raw.append("# " + line + "\n") |
|
2080 | 2155 | more = self.push(self.prefilter(line,more)) |
|
2081 | 2156 | # IPython's runsource returns None if there was an error |
|
2082 | 2157 | # compiling the code. This allows us to stop processing right |
|
2083 | 2158 | # away, so the user gets the error message at the right place. |
|
2084 | 2159 | if more is None: |
|
2085 | 2160 | break |
|
2086 | 2161 | else: |
|
2087 | 2162 | self.input_hist_raw.append("\n") |
|
2088 | 2163 | # final newline in case the input didn't have it, so that the code |
|
2089 | 2164 | # actually does get executed |
|
2090 | 2165 | if more: |
|
2091 | 2166 | self.push('\n') |
|
2092 | 2167 | |
|
2093 | 2168 | def runsource(self, source, filename='<input>', symbol='single'): |
|
2094 | 2169 | """Compile and run some source in the interpreter. |
|
2095 | 2170 | |
|
2096 | 2171 | Arguments are as for compile_command(). |
|
2097 | 2172 | |
|
2098 | 2173 | One several things can happen: |
|
2099 | 2174 | |
|
2100 | 2175 | 1) The input is incorrect; compile_command() raised an |
|
2101 | 2176 | exception (SyntaxError or OverflowError). A syntax traceback |
|
2102 | 2177 | will be printed by calling the showsyntaxerror() method. |
|
2103 | 2178 | |
|
2104 | 2179 | 2) The input is incomplete, and more input is required; |
|
2105 | 2180 | compile_command() returned None. Nothing happens. |
|
2106 | 2181 | |
|
2107 | 2182 | 3) The input is complete; compile_command() returned a code |
|
2108 | 2183 | object. The code is executed by calling self.runcode() (which |
|
2109 | 2184 | also handles run-time exceptions, except for SystemExit). |
|
2110 | 2185 | |
|
2111 | 2186 | The return value is: |
|
2112 | 2187 | |
|
2113 | 2188 | - True in case 2 |
|
2114 | 2189 | |
|
2115 | 2190 | - False in the other cases, unless an exception is raised, where |
|
2116 | 2191 | None is returned instead. This can be used by external callers to |
|
2117 | 2192 | know whether to continue feeding input or not. |
|
2118 | 2193 | |
|
2119 | 2194 | The return value can be used to decide whether to use sys.ps1 or |
|
2120 | 2195 | sys.ps2 to prompt the next line.""" |
|
2121 | 2196 | |
|
2122 | 2197 | # if the source code has leading blanks, add 'if 1:\n' to it |
|
2123 | 2198 | # this allows execution of indented pasted code. It is tempting |
|
2124 | 2199 | # to add '\n' at the end of source to run commands like ' a=1' |
|
2125 | 2200 | # directly, but this fails for more complicated scenarios |
|
2126 | 2201 | source=source.encode(self.stdin_encoding) |
|
2127 | 2202 | if source[:1] in [' ', '\t']: |
|
2128 | 2203 | source = 'if 1:\n%s' % source |
|
2129 | 2204 | |
|
2130 | 2205 | try: |
|
2131 | 2206 | code = self.compile(source,filename,symbol) |
|
2132 | 2207 | except (OverflowError, SyntaxError, ValueError, TypeError): |
|
2133 | 2208 | # Case 1 |
|
2134 | 2209 | self.showsyntaxerror(filename) |
|
2135 | 2210 | return None |
|
2136 | 2211 | |
|
2137 | 2212 | if code is None: |
|
2138 | 2213 | # Case 2 |
|
2139 | 2214 | return True |
|
2140 | 2215 | |
|
2141 | 2216 | # Case 3 |
|
2142 | 2217 | # We store the code object so that threaded shells and |
|
2143 | 2218 | # custom exception handlers can access all this info if needed. |
|
2144 | 2219 | # The source corresponding to this can be obtained from the |
|
2145 | 2220 | # buffer attribute as '\n'.join(self.buffer). |
|
2146 | 2221 | self.code_to_run = code |
|
2147 | 2222 | # now actually execute the code object |
|
2148 | 2223 | if self.runcode(code) == 0: |
|
2149 | 2224 | return False |
|
2150 | 2225 | else: |
|
2151 | 2226 | return None |
|
2152 | 2227 | |
|
2153 | 2228 | def runcode(self,code_obj): |
|
2154 | 2229 | """Execute a code object. |
|
2155 | 2230 | |
|
2156 | 2231 | When an exception occurs, self.showtraceback() is called to display a |
|
2157 | 2232 | traceback. |
|
2158 | 2233 | |
|
2159 | 2234 | Return value: a flag indicating whether the code to be run completed |
|
2160 | 2235 | successfully: |
|
2161 | 2236 | |
|
2162 | 2237 | - 0: successful execution. |
|
2163 | 2238 | - 1: an error occurred. |
|
2164 | 2239 | """ |
|
2165 | 2240 | |
|
2166 | 2241 | # Set our own excepthook in case the user code tries to call it |
|
2167 | 2242 | # directly, so that the IPython crash handler doesn't get triggered |
|
2168 | 2243 | old_excepthook,sys.excepthook = sys.excepthook, self.excepthook |
|
2169 | 2244 | |
|
2170 | 2245 | # we save the original sys.excepthook in the instance, in case config |
|
2171 | 2246 | # code (such as magics) needs access to it. |
|
2172 | 2247 | self.sys_excepthook = old_excepthook |
|
2173 | 2248 | outflag = 1 # happens in more places, so it's easier as default |
|
2174 | 2249 | try: |
|
2175 | 2250 | try: |
|
2176 | 2251 | self.hooks.pre_runcode_hook() |
|
2177 | 2252 | exec code_obj in self.user_global_ns, self.user_ns |
|
2178 | 2253 | finally: |
|
2179 | 2254 | # Reset our crash handler in place |
|
2180 | 2255 | sys.excepthook = old_excepthook |
|
2181 | 2256 | except SystemExit: |
|
2182 | 2257 | self.resetbuffer() |
|
2183 | 2258 | self.showtraceback() |
|
2184 | 2259 | warn("Type %exit or %quit to exit IPython " |
|
2185 | 2260 | "(%Exit or %Quit do so unconditionally).",level=1) |
|
2186 | 2261 | except self.custom_exceptions: |
|
2187 | 2262 | etype,value,tb = sys.exc_info() |
|
2188 | 2263 | self.CustomTB(etype,value,tb) |
|
2189 | 2264 | except: |
|
2190 | 2265 | self.showtraceback() |
|
2191 | 2266 | else: |
|
2192 | 2267 | outflag = 0 |
|
2193 | 2268 | if softspace(sys.stdout, 0): |
|
2194 | 2269 | |
|
2195 | 2270 | # Flush out code object which has been run (and source) |
|
2196 | 2271 | self.code_to_run = None |
|
2197 | 2272 | return outflag |
|
2198 | 2273 | |
|
2199 | 2274 | def push(self, line): |
|
2200 | 2275 | """Push a line to the interpreter. |
|
2201 | 2276 | |
|
2202 | 2277 | The line should not have a trailing newline; it may have |
|
2203 | 2278 | internal newlines. The line is appended to a buffer and the |
|
2204 | 2279 | interpreter's runsource() method is called with the |
|
2205 | 2280 | concatenated contents of the buffer as source. If this |
|
2206 | 2281 | indicates that the command was executed or invalid, the buffer |
|
2207 | 2282 | is reset; otherwise, the command is incomplete, and the buffer |
|
2208 | 2283 | is left as it was after the line was appended. The return |
|
2209 | 2284 | value is 1 if more input is required, 0 if the line was dealt |
|
2210 | 2285 | with in some way (this is the same as runsource()). |
|
2211 | 2286 | """ |
|
2212 | 2287 | |
|
2213 | 2288 | # autoindent management should be done here, and not in the |
|
2214 | 2289 | # interactive loop, since that one is only seen by keyboard input. We |
|
2215 | 2290 | # need this done correctly even for code run via runlines (which uses |
|
2216 | 2291 | # push). |
|
2217 | 2292 | |
|
2218 | 2293 | #print 'push line: <%s>' % line # dbg |
|
2219 | 2294 | for subline in line.splitlines(): |
|
2220 | 2295 | self.autoindent_update(subline) |
|
2221 | 2296 | self.buffer.append(line) |
|
2222 | 2297 | more = self.runsource('\n'.join(self.buffer), self.filename) |
|
2223 | 2298 | if not more: |
|
2224 | 2299 | self.resetbuffer() |
|
2225 | 2300 | return more |
|
2226 | 2301 | |
|
2227 | 2302 | def split_user_input(self, line): |
|
2228 | 2303 | # This is really a hold-over to support ipapi and some extensions |
|
2229 | 2304 | return prefilter.splitUserInput(line) |
|
2230 | 2305 | |
|
2231 | 2306 | def resetbuffer(self): |
|
2232 | 2307 | """Reset the input buffer.""" |
|
2233 | 2308 | self.buffer[:] = [] |
|
2234 | 2309 | |
|
2235 | 2310 | def raw_input(self,prompt='',continue_prompt=False): |
|
2236 | 2311 | """Write a prompt and read a line. |
|
2237 | 2312 | |
|
2238 | 2313 | The returned line does not include the trailing newline. |
|
2239 | 2314 | When the user enters the EOF key sequence, EOFError is raised. |
|
2240 | 2315 | |
|
2241 | 2316 | Optional inputs: |
|
2242 | 2317 | |
|
2243 | 2318 | - prompt(''): a string to be printed to prompt the user. |
|
2244 | 2319 | |
|
2245 | 2320 | - continue_prompt(False): whether this line is the first one or a |
|
2246 | 2321 | continuation in a sequence of inputs. |
|
2247 | 2322 | """ |
|
2248 | 2323 | |
|
2249 | 2324 | # Code run by the user may have modified the readline completer state. |
|
2250 | 2325 | # We must ensure that our completer is back in place. |
|
2251 | 2326 | if self.has_readline: |
|
2252 | 2327 | self.set_completer() |
|
2253 | 2328 | |
|
2254 | 2329 | try: |
|
2255 | 2330 | line = raw_input_original(prompt).decode(self.stdin_encoding) |
|
2256 | 2331 | except ValueError: |
|
2257 | 2332 | warn("\n********\nYou or a %run:ed script called sys.stdin.close()" |
|
2258 | 2333 | " or sys.stdout.close()!\nExiting IPython!") |
|
2259 | 2334 | self.ask_exit() |
|
2260 | 2335 | return "" |
|
2261 | 2336 | |
|
2262 | 2337 | # Try to be reasonably smart about not re-indenting pasted input more |
|
2263 | 2338 | # than necessary. We do this by trimming out the auto-indent initial |
|
2264 | 2339 | # spaces, if the user's actual input started itself with whitespace. |
|
2265 | 2340 | #debugx('self.buffer[-1]') |
|
2266 | 2341 | |
|
2267 | 2342 | if self.autoindent: |
|
2268 | 2343 | if num_ini_spaces(line) > self.indent_current_nsp: |
|
2269 | 2344 | line = line[self.indent_current_nsp:] |
|
2270 | 2345 | self.indent_current_nsp = 0 |
|
2271 | 2346 | |
|
2272 | 2347 | # store the unfiltered input before the user has any chance to modify |
|
2273 | 2348 | # it. |
|
2274 | 2349 | if line.strip(): |
|
2275 | 2350 | if continue_prompt: |
|
2276 | 2351 | self.input_hist_raw[-1] += '%s\n' % line |
|
2277 | 2352 | if self.has_readline: # and some config option is set? |
|
2278 | 2353 | try: |
|
2279 | 2354 | histlen = self.readline.get_current_history_length() |
|
2280 | 2355 | if histlen > 1: |
|
2281 | 2356 | newhist = self.input_hist_raw[-1].rstrip() |
|
2282 | 2357 | self.readline.remove_history_item(histlen-1) |
|
2283 | 2358 | self.readline.replace_history_item(histlen-2, |
|
2284 | 2359 | newhist.encode(self.stdin_encoding)) |
|
2285 | 2360 | except AttributeError: |
|
2286 | 2361 | pass # re{move,place}_history_item are new in 2.4. |
|
2287 | 2362 | else: |
|
2288 | 2363 | self.input_hist_raw.append('%s\n' % line) |
|
2289 | 2364 | # only entries starting at first column go to shadow history |
|
2290 | 2365 | if line.lstrip() == line: |
|
2291 | 2366 | self.shadowhist.add(line.strip()) |
|
2292 | 2367 | elif not continue_prompt: |
|
2293 | 2368 | self.input_hist_raw.append('\n') |
|
2294 | 2369 | try: |
|
2295 | 2370 | lineout = self.prefilter(line,continue_prompt) |
|
2296 | 2371 | except: |
|
2297 | 2372 | # blanket except, in case a user-defined prefilter crashes, so it |
|
2298 | 2373 | # can't take all of ipython with it. |
|
2299 | 2374 | self.showtraceback() |
|
2300 | 2375 | return '' |
|
2301 | 2376 | else: |
|
2302 | 2377 | return lineout |
|
2303 | 2378 | |
|
2304 | 2379 | def _prefilter(self, line, continue_prompt): |
|
2305 | 2380 | """Calls different preprocessors, depending on the form of line.""" |
|
2306 | 2381 | |
|
2307 | 2382 | # All handlers *must* return a value, even if it's blank (''). |
|
2308 | 2383 | |
|
2309 | 2384 | # Lines are NOT logged here. Handlers should process the line as |
|
2310 | 2385 | # needed, update the cache AND log it (so that the input cache array |
|
2311 | 2386 | # stays synced). |
|
2312 | 2387 | |
|
2313 | 2388 | #..................................................................... |
|
2314 | 2389 | # Code begins |
|
2315 | 2390 | |
|
2316 | 2391 | #if line.startswith('%crash'): raise RuntimeError,'Crash now!' # dbg |
|
2317 | 2392 | |
|
2318 | 2393 | # save the line away in case we crash, so the post-mortem handler can |
|
2319 | 2394 | # record it |
|
2320 | 2395 | self._last_input_line = line |
|
2321 | 2396 | |
|
2322 | 2397 | #print '***line: <%s>' % line # dbg |
|
2323 | 2398 | |
|
2324 | 2399 | if not line: |
|
2325 | 2400 | # Return immediately on purely empty lines, so that if the user |
|
2326 | 2401 | # previously typed some whitespace that started a continuation |
|
2327 | 2402 | # prompt, he can break out of that loop with just an empty line. |
|
2328 | 2403 | # This is how the default python prompt works. |
|
2329 | 2404 | |
|
2330 | 2405 | # Only return if the accumulated input buffer was just whitespace! |
|
2331 | 2406 | if ''.join(self.buffer).isspace(): |
|
2332 | 2407 | self.buffer[:] = [] |
|
2333 | 2408 | return '' |
|
2334 | 2409 | |
|
2335 | 2410 | line_info = prefilter.LineInfo(line, continue_prompt) |
|
2336 | 2411 | |
|
2337 | 2412 | # the input history needs to track even empty lines |
|
2338 | 2413 | stripped = line.strip() |
|
2339 | 2414 | |
|
2340 | 2415 | if not stripped: |
|
2341 | 2416 | if not continue_prompt: |
|
2342 | 2417 | self.outputcache.prompt_count -= 1 |
|
2343 | 2418 | return self.handle_normal(line_info) |
|
2344 | 2419 | |
|
2345 | 2420 | # print '***cont',continue_prompt # dbg |
|
2346 | 2421 | # special handlers are only allowed for single line statements |
|
2347 | 2422 | if continue_prompt and not self.rc.multi_line_specials: |
|
2348 | 2423 | return self.handle_normal(line_info) |
|
2349 | 2424 | |
|
2350 | 2425 | |
|
2351 | 2426 | # See whether any pre-existing handler can take care of it |
|
2352 | 2427 | rewritten = self.hooks.input_prefilter(stripped) |
|
2353 | 2428 | if rewritten != stripped: # ok, some prefilter did something |
|
2354 | 2429 | rewritten = line_info.pre + rewritten # add indentation |
|
2355 | 2430 | return self.handle_normal(prefilter.LineInfo(rewritten, |
|
2356 | 2431 | continue_prompt)) |
|
2357 | 2432 | |
|
2358 | 2433 | #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun,theRest) # dbg |
|
2359 | 2434 | |
|
2360 | 2435 | return prefilter.prefilter(line_info, self) |
|
2361 | 2436 | |
|
2362 | 2437 | |
|
2363 | 2438 | def _prefilter_dumb(self, line, continue_prompt): |
|
2364 | 2439 | """simple prefilter function, for debugging""" |
|
2365 | 2440 | return self.handle_normal(line,continue_prompt) |
|
2366 | 2441 | |
|
2367 | 2442 | |
|
2368 | 2443 | def multiline_prefilter(self, line, continue_prompt): |
|
2369 | 2444 | """ Run _prefilter for each line of input |
|
2370 | 2445 | |
|
2371 | 2446 | Covers cases where there are multiple lines in the user entry, |
|
2372 | 2447 | which is the case when the user goes back to a multiline history |
|
2373 | 2448 | entry and presses enter. |
|
2374 | 2449 | |
|
2375 | 2450 | """ |
|
2376 | 2451 | out = [] |
|
2377 | 2452 | for l in line.rstrip('\n').split('\n'): |
|
2378 | 2453 | out.append(self._prefilter(l, continue_prompt)) |
|
2379 | 2454 | return '\n'.join(out) |
|
2380 | 2455 | |
|
2381 | 2456 | # Set the default prefilter() function (this can be user-overridden) |
|
2382 | 2457 | prefilter = multiline_prefilter |
|
2383 | 2458 | |
|
2384 | 2459 | def handle_normal(self,line_info): |
|
2385 | 2460 | """Handle normal input lines. Use as a template for handlers.""" |
|
2386 | 2461 | |
|
2387 | 2462 | # With autoindent on, we need some way to exit the input loop, and I |
|
2388 | 2463 | # don't want to force the user to have to backspace all the way to |
|
2389 | 2464 | # clear the line. The rule will be in this case, that either two |
|
2390 | 2465 | # lines of pure whitespace in a row, or a line of pure whitespace but |
|
2391 | 2466 | # of a size different to the indent level, will exit the input loop. |
|
2392 | 2467 | line = line_info.line |
|
2393 | 2468 | continue_prompt = line_info.continue_prompt |
|
2394 | 2469 | |
|
2395 | 2470 | if (continue_prompt and self.autoindent and line.isspace() and |
|
2396 | 2471 | (0 < abs(len(line) - self.indent_current_nsp) <= 2 or |
|
2397 | 2472 | (self.buffer[-1]).isspace() )): |
|
2398 | 2473 | line = '' |
|
2399 | 2474 | |
|
2400 | 2475 | self.log(line,line,continue_prompt) |
|
2401 | 2476 | return line |
|
2402 | 2477 | |
|
2403 | 2478 | def handle_alias(self,line_info): |
|
2404 | 2479 | """Handle alias input lines. """ |
|
2405 | 2480 | tgt = self.alias_table[line_info.iFun] |
|
2406 | 2481 | # print "=>",tgt #dbg |
|
2407 | 2482 | if callable(tgt): |
|
2408 | 2483 | if '$' in line_info.line: |
|
2409 | 2484 | call_meth = '(_ip, _ip.itpl(%s))' |
|
2410 | 2485 | else: |
|
2411 | 2486 | call_meth = '(_ip,%s)' |
|
2412 | 2487 | line_out = ("%s_sh.%s" + call_meth) % (line_info.preWhitespace, |
|
2413 | 2488 | line_info.iFun, |
|
2414 | 2489 | make_quoted_expr(line_info.line)) |
|
2415 | 2490 | else: |
|
2416 | 2491 | transformed = self.expand_aliases(line_info.iFun,line_info.theRest) |
|
2417 | 2492 | |
|
2418 | 2493 | # pre is needed, because it carries the leading whitespace. Otherwise |
|
2419 | 2494 | # aliases won't work in indented sections. |
|
2420 | 2495 | line_out = '%s_ip.system(%s)' % (line_info.preWhitespace, |
|
2421 | 2496 | make_quoted_expr( transformed )) |
|
2422 | 2497 | |
|
2423 | 2498 | self.log(line_info.line,line_out,line_info.continue_prompt) |
|
2424 | 2499 | #print 'line out:',line_out # dbg |
|
2425 | 2500 | return line_out |
|
2426 | 2501 | |
|
2427 | 2502 | def handle_shell_escape(self, line_info): |
|
2428 | 2503 | """Execute the line in a shell, empty return value""" |
|
2429 | 2504 | #print 'line in :', `line` # dbg |
|
2430 | 2505 | line = line_info.line |
|
2431 | 2506 | if line.lstrip().startswith('!!'): |
|
2432 | 2507 | # rewrite LineInfo's line, iFun and theRest to properly hold the |
|
2433 | 2508 | # call to %sx and the actual command to be executed, so |
|
2434 | 2509 | # handle_magic can work correctly. Note that this works even if |
|
2435 | 2510 | # the line is indented, so it handles multi_line_specials |
|
2436 | 2511 | # properly. |
|
2437 | 2512 | new_rest = line.lstrip()[2:] |
|
2438 | 2513 | line_info.line = '%ssx %s' % (self.ESC_MAGIC,new_rest) |
|
2439 | 2514 | line_info.iFun = 'sx' |
|
2440 | 2515 | line_info.theRest = new_rest |
|
2441 | 2516 | return self.handle_magic(line_info) |
|
2442 | 2517 | else: |
|
2443 | 2518 | cmd = line.lstrip().lstrip('!') |
|
2444 | 2519 | line_out = '%s_ip.system(%s)' % (line_info.preWhitespace, |
|
2445 | 2520 | make_quoted_expr(cmd)) |
|
2446 | 2521 | # update cache/log and return |
|
2447 | 2522 | self.log(line,line_out,line_info.continue_prompt) |
|
2448 | 2523 | return line_out |
|
2449 | 2524 | |
|
2450 | 2525 | def handle_magic(self, line_info): |
|
2451 | 2526 | """Execute magic functions.""" |
|
2452 | 2527 | iFun = line_info.iFun |
|
2453 | 2528 | theRest = line_info.theRest |
|
2454 | 2529 | cmd = '%s_ip.magic(%s)' % (line_info.preWhitespace, |
|
2455 | 2530 | make_quoted_expr(iFun + " " + theRest)) |
|
2456 | 2531 | self.log(line_info.line,cmd,line_info.continue_prompt) |
|
2457 | 2532 | #print 'in handle_magic, cmd=<%s>' % cmd # dbg |
|
2458 | 2533 | return cmd |
|
2459 | 2534 | |
|
2460 | 2535 | def handle_auto(self, line_info): |
|
2461 | 2536 | """Hande lines which can be auto-executed, quoting if requested.""" |
|
2462 | 2537 | |
|
2463 | 2538 | line = line_info.line |
|
2464 | 2539 | iFun = line_info.iFun |
|
2465 | 2540 | theRest = line_info.theRest |
|
2466 | 2541 | pre = line_info.pre |
|
2467 | 2542 | continue_prompt = line_info.continue_prompt |
|
2468 | 2543 | obj = line_info.ofind(self)['obj'] |
|
2469 | 2544 | |
|
2470 | 2545 | #print 'pre <%s> iFun <%s> rest <%s>' % (pre,iFun,theRest) # dbg |
|
2471 | 2546 | |
|
2472 | 2547 | # This should only be active for single-line input! |
|
2473 | 2548 | if continue_prompt: |
|
2474 | 2549 | self.log(line,line,continue_prompt) |
|
2475 | 2550 | return line |
|
2476 | 2551 | |
|
2477 | 2552 | force_auto = isinstance(obj, IPython.ipapi.IPyAutocall) |
|
2478 | 2553 | auto_rewrite = True |
|
2479 | 2554 | |
|
2480 | 2555 | if pre == self.ESC_QUOTE: |
|
2481 | 2556 | # Auto-quote splitting on whitespace |
|
2482 | 2557 | newcmd = '%s("%s")' % (iFun,'", "'.join(theRest.split()) ) |
|
2483 | 2558 | elif pre == self.ESC_QUOTE2: |
|
2484 | 2559 | # Auto-quote whole string |
|
2485 | 2560 | newcmd = '%s("%s")' % (iFun,theRest) |
|
2486 | 2561 | elif pre == self.ESC_PAREN: |
|
2487 | 2562 | newcmd = '%s(%s)' % (iFun,",".join(theRest.split())) |
|
2488 | 2563 | else: |
|
2489 | 2564 | # Auto-paren. |
|
2490 | 2565 | # We only apply it to argument-less calls if the autocall |
|
2491 | 2566 | # parameter is set to 2. We only need to check that autocall is < |
|
2492 | 2567 | # 2, since this function isn't called unless it's at least 1. |
|
2493 | 2568 | if not theRest and (self.rc.autocall < 2) and not force_auto: |
|
2494 | 2569 | newcmd = '%s %s' % (iFun,theRest) |
|
2495 | 2570 | auto_rewrite = False |
|
2496 | 2571 | else: |
|
2497 | 2572 | if not force_auto and theRest.startswith('['): |
|
2498 | 2573 | if hasattr(obj,'__getitem__'): |
|
2499 | 2574 | # Don't autocall in this case: item access for an object |
|
2500 | 2575 | # which is BOTH callable and implements __getitem__. |
|
2501 | 2576 | newcmd = '%s %s' % (iFun,theRest) |
|
2502 | 2577 | auto_rewrite = False |
|
2503 | 2578 | else: |
|
2504 | 2579 | # if the object doesn't support [] access, go ahead and |
|
2505 | 2580 | # autocall |
|
2506 | 2581 | newcmd = '%s(%s)' % (iFun.rstrip(),theRest) |
|
2507 | 2582 | elif theRest.endswith(';'): |
|
2508 | 2583 | newcmd = '%s(%s);' % (iFun.rstrip(),theRest[:-1]) |
|
2509 | 2584 | else: |
|
2510 | 2585 | newcmd = '%s(%s)' % (iFun.rstrip(), theRest) |
|
2511 | 2586 | |
|
2512 | 2587 | if auto_rewrite: |
|
2513 | 2588 | rw = self.outputcache.prompt1.auto_rewrite() + newcmd |
|
2514 | 2589 | |
|
2515 | 2590 | try: |
|
2516 | 2591 | # plain ascii works better w/ pyreadline, on some machines, so |
|
2517 | 2592 | # we use it and only print uncolored rewrite if we have unicode |
|
2518 | 2593 | rw = str(rw) |
|
2519 | 2594 | print >>Term.cout, rw |
|
2520 | 2595 | except UnicodeEncodeError: |
|
2521 | 2596 | print "-------------->" + newcmd |
|
2522 | 2597 | |
|
2523 | 2598 | # log what is now valid Python, not the actual user input (without the |
|
2524 | 2599 | # final newline) |
|
2525 | 2600 | self.log(line,newcmd,continue_prompt) |
|
2526 | 2601 | return newcmd |
|
2527 | 2602 | |
|
2528 | 2603 | def handle_help(self, line_info): |
|
2529 | 2604 | """Try to get some help for the object. |
|
2530 | 2605 | |
|
2531 | 2606 | obj? or ?obj -> basic information. |
|
2532 | 2607 | obj?? or ??obj -> more details. |
|
2533 | 2608 | """ |
|
2534 | 2609 | |
|
2535 | 2610 | line = line_info.line |
|
2536 | 2611 | # We need to make sure that we don't process lines which would be |
|
2537 | 2612 | # otherwise valid python, such as "x=1 # what?" |
|
2538 | 2613 | try: |
|
2539 | 2614 | codeop.compile_command(line) |
|
2540 | 2615 | except SyntaxError: |
|
2541 | 2616 | # We should only handle as help stuff which is NOT valid syntax |
|
2542 | 2617 | if line[0]==self.ESC_HELP: |
|
2543 | 2618 | line = line[1:] |
|
2544 | 2619 | elif line[-1]==self.ESC_HELP: |
|
2545 | 2620 | line = line[:-1] |
|
2546 | 2621 | self.log(line,'#?'+line,line_info.continue_prompt) |
|
2547 | 2622 | if line: |
|
2548 | 2623 | #print 'line:<%r>' % line # dbg |
|
2549 | 2624 | self.magic_pinfo(line) |
|
2550 | 2625 | else: |
|
2551 | 2626 | page(self.usage,screen_lines=self.rc.screen_length) |
|
2552 | 2627 | return '' # Empty string is needed here! |
|
2553 | 2628 | except: |
|
2554 | 2629 | # Pass any other exceptions through to the normal handler |
|
2555 | 2630 | return self.handle_normal(line_info) |
|
2556 | 2631 | else: |
|
2557 | 2632 | # If the code compiles ok, we should handle it normally |
|
2558 | 2633 | return self.handle_normal(line_info) |
|
2559 | 2634 | |
|
2560 | 2635 | def getapi(self): |
|
2561 | 2636 | """ Get an IPApi object for this shell instance |
|
2562 | 2637 | |
|
2563 | 2638 | Getting an IPApi object is always preferable to accessing the shell |
|
2564 | 2639 | directly, but this holds true especially for extensions. |
|
2565 | 2640 | |
|
2566 | 2641 | It should always be possible to implement an extension with IPApi |
|
2567 | 2642 | alone. If not, contact maintainer to request an addition. |
|
2568 | 2643 | |
|
2569 | 2644 | """ |
|
2570 | 2645 | return self.api |
|
2571 | 2646 | |
|
2572 | 2647 | def handle_emacs(self, line_info): |
|
2573 | 2648 | """Handle input lines marked by python-mode.""" |
|
2574 | 2649 | |
|
2575 | 2650 | # Currently, nothing is done. Later more functionality can be added |
|
2576 | 2651 | # here if needed. |
|
2577 | 2652 | |
|
2578 | 2653 | # The input cache shouldn't be updated |
|
2579 | 2654 | return line_info.line |
|
2580 | 2655 | |
|
2581 | 2656 | |
|
2582 | 2657 | def mktempfile(self,data=None): |
|
2583 | 2658 | """Make a new tempfile and return its filename. |
|
2584 | 2659 | |
|
2585 | 2660 | This makes a call to tempfile.mktemp, but it registers the created |
|
2586 | 2661 | filename internally so ipython cleans it up at exit time. |
|
2587 | 2662 | |
|
2588 | 2663 | Optional inputs: |
|
2589 | 2664 | |
|
2590 | 2665 | - data(None): if data is given, it gets written out to the temp file |
|
2591 | 2666 | immediately, and the file is closed again.""" |
|
2592 | 2667 | |
|
2593 | 2668 | filename = tempfile.mktemp('.py','ipython_edit_') |
|
2594 | 2669 | self.tempfiles.append(filename) |
|
2595 | 2670 | |
|
2596 | 2671 | if data: |
|
2597 | 2672 | tmp_file = open(filename,'w') |
|
2598 | 2673 | tmp_file.write(data) |
|
2599 | 2674 | tmp_file.close() |
|
2600 | 2675 | return filename |
|
2601 | 2676 | |
|
2602 | 2677 | def write(self,data): |
|
2603 | 2678 | """Write a string to the default output""" |
|
2604 | 2679 | Term.cout.write(data) |
|
2605 | 2680 | |
|
2606 | 2681 | def write_err(self,data): |
|
2607 | 2682 | """Write a string to the default error output""" |
|
2608 | 2683 | Term.cerr.write(data) |
|
2609 | 2684 | |
|
2610 | 2685 | def ask_exit(self): |
|
2611 | 2686 | """ Call for exiting. Can be overiden and used as a callback. """ |
|
2612 | 2687 | self.exit_now = True |
|
2613 | 2688 | |
|
2614 | 2689 | def exit(self): |
|
2615 | 2690 | """Handle interactive exit. |
|
2616 | 2691 | |
|
2617 | 2692 | This method calls the ask_exit callback.""" |
|
2618 | 2693 | |
|
2619 | 2694 | if self.rc.confirm_exit: |
|
2620 | 2695 | if self.ask_yes_no('Do you really want to exit ([y]/n)?','y'): |
|
2621 | 2696 | self.ask_exit() |
|
2622 | 2697 | else: |
|
2623 | 2698 | self.ask_exit() |
|
2624 | 2699 | |
|
2625 | 2700 | def safe_execfile(self,fname,*where,**kw): |
|
2626 | 2701 | """A safe version of the builtin execfile(). |
|
2627 | 2702 | |
|
2628 | 2703 | This version will never throw an exception, and knows how to handle |
|
2629 | 2704 | ipython logs as well. |
|
2630 | 2705 | |
|
2631 | 2706 | :Parameters: |
|
2632 | 2707 | fname : string |
|
2633 | 2708 | Name of the file to be executed. |
|
2634 | 2709 | |
|
2635 | 2710 | where : tuple |
|
2636 | 2711 | One or two namespaces, passed to execfile() as (globals,locals). |
|
2637 | 2712 | If only one is given, it is passed as both. |
|
2638 | 2713 | |
|
2639 | 2714 | :Keywords: |
|
2640 | 2715 | islog : boolean (False) |
|
2641 | 2716 | |
|
2642 | 2717 | quiet : boolean (True) |
|
2643 | 2718 | |
|
2644 | 2719 | exit_ignore : boolean (False) |
|
2645 | 2720 | """ |
|
2646 | 2721 | |
|
2647 | 2722 | def syspath_cleanup(): |
|
2648 | 2723 | """Internal cleanup routine for sys.path.""" |
|
2649 | 2724 | if add_dname: |
|
2650 | 2725 | try: |
|
2651 | 2726 | sys.path.remove(dname) |
|
2652 | 2727 | except ValueError: |
|
2653 | 2728 | # For some reason the user has already removed it, ignore. |
|
2654 | 2729 | pass |
|
2655 | 2730 | |
|
2656 | 2731 | fname = os.path.expanduser(fname) |
|
2657 | 2732 | |
|
2658 | 2733 | # Find things also in current directory. This is needed to mimic the |
|
2659 | 2734 | # behavior of running a script from the system command line, where |
|
2660 | 2735 | # Python inserts the script's directory into sys.path |
|
2661 | 2736 | dname = os.path.dirname(os.path.abspath(fname)) |
|
2662 | 2737 | add_dname = False |
|
2663 | 2738 | if dname not in sys.path: |
|
2664 | 2739 | sys.path.insert(0,dname) |
|
2665 | 2740 | add_dname = True |
|
2666 | 2741 | |
|
2667 | 2742 | try: |
|
2668 | 2743 | xfile = open(fname) |
|
2669 | 2744 | except: |
|
2670 | 2745 | print >> Term.cerr, \ |
|
2671 | 2746 | 'Could not open file <%s> for safe execution.' % fname |
|
2672 | 2747 | syspath_cleanup() |
|
2673 | 2748 | return None |
|
2674 | 2749 | |
|
2675 | 2750 | kw.setdefault('islog',0) |
|
2676 | 2751 | kw.setdefault('quiet',1) |
|
2677 | 2752 | kw.setdefault('exit_ignore',0) |
|
2678 | 2753 | |
|
2679 | 2754 | first = xfile.readline() |
|
2680 | 2755 | loghead = str(self.loghead_tpl).split('\n',1)[0].strip() |
|
2681 | 2756 | xfile.close() |
|
2682 | 2757 | # line by line execution |
|
2683 | 2758 | if first.startswith(loghead) or kw['islog']: |
|
2684 | 2759 | print 'Loading log file <%s> one line at a time...' % fname |
|
2685 | 2760 | if kw['quiet']: |
|
2686 | 2761 | stdout_save = sys.stdout |
|
2687 | 2762 | sys.stdout = StringIO.StringIO() |
|
2688 | 2763 | try: |
|
2689 | 2764 | globs,locs = where[0:2] |
|
2690 | 2765 | except: |
|
2691 | 2766 | try: |
|
2692 | 2767 | globs = locs = where[0] |
|
2693 | 2768 | except: |
|
2694 | 2769 | globs = locs = globals() |
|
2695 | 2770 | badblocks = [] |
|
2696 | 2771 | |
|
2697 | 2772 | # we also need to identify indented blocks of code when replaying |
|
2698 | 2773 | # logs and put them together before passing them to an exec |
|
2699 | 2774 | # statement. This takes a bit of regexp and look-ahead work in the |
|
2700 | 2775 | # file. It's easiest if we swallow the whole thing in memory |
|
2701 | 2776 | # first, and manually walk through the lines list moving the |
|
2702 | 2777 | # counter ourselves. |
|
2703 | 2778 | indent_re = re.compile('\s+\S') |
|
2704 | 2779 | xfile = open(fname) |
|
2705 | 2780 | filelines = xfile.readlines() |
|
2706 | 2781 | xfile.close() |
|
2707 | 2782 | nlines = len(filelines) |
|
2708 | 2783 | lnum = 0 |
|
2709 | 2784 | while lnum < nlines: |
|
2710 | 2785 | line = filelines[lnum] |
|
2711 | 2786 | lnum += 1 |
|
2712 | 2787 | # don't re-insert logger status info into cache |
|
2713 | 2788 | if line.startswith('#log#'): |
|
2714 | 2789 | continue |
|
2715 | 2790 | else: |
|
2716 | 2791 | # build a block of code (maybe a single line) for execution |
|
2717 | 2792 | block = line |
|
2718 | 2793 | try: |
|
2719 | 2794 | next = filelines[lnum] # lnum has already incremented |
|
2720 | 2795 | except: |
|
2721 | 2796 | next = None |
|
2722 | 2797 | while next and indent_re.match(next): |
|
2723 | 2798 | block += next |
|
2724 | 2799 | lnum += 1 |
|
2725 | 2800 | try: |
|
2726 | 2801 | next = filelines[lnum] |
|
2727 | 2802 | except: |
|
2728 | 2803 | next = None |
|
2729 | 2804 | # now execute the block of one or more lines |
|
2730 | 2805 | try: |
|
2731 | 2806 | exec block in globs,locs |
|
2732 | 2807 | except SystemExit: |
|
2733 | 2808 | pass |
|
2734 | 2809 | except: |
|
2735 | 2810 | badblocks.append(block.rstrip()) |
|
2736 | 2811 | if kw['quiet']: # restore stdout |
|
2737 | 2812 | sys.stdout.close() |
|
2738 | 2813 | sys.stdout = stdout_save |
|
2739 | 2814 | print 'Finished replaying log file <%s>' % fname |
|
2740 | 2815 | if badblocks: |
|
2741 | 2816 | print >> sys.stderr, ('\nThe following lines/blocks in file ' |
|
2742 | 2817 | '<%s> reported errors:' % fname) |
|
2743 | 2818 | |
|
2744 | 2819 | for badline in badblocks: |
|
2745 | 2820 | print >> sys.stderr, badline |
|
2746 | 2821 | else: # regular file execution |
|
2747 | 2822 | try: |
|
2748 | 2823 | if sys.platform == 'win32' and sys.version_info < (2,5,1): |
|
2749 | 2824 | # Work around a bug in Python for Windows. The bug was |
|
2750 | 2825 | # fixed in in Python 2.5 r54159 and 54158, but that's still |
|
2751 | 2826 | # SVN Python as of March/07. For details, see: |
|
2752 | 2827 | # http://projects.scipy.org/ipython/ipython/ticket/123 |
|
2753 | 2828 | try: |
|
2754 | 2829 | globs,locs = where[0:2] |
|
2755 | 2830 | except: |
|
2756 | 2831 | try: |
|
2757 | 2832 | globs = locs = where[0] |
|
2758 | 2833 | except: |
|
2759 | 2834 | globs = locs = globals() |
|
2760 | 2835 | exec file(fname) in globs,locs |
|
2761 | 2836 | else: |
|
2762 | 2837 | execfile(fname,*where) |
|
2763 | 2838 | except SyntaxError: |
|
2764 | 2839 | self.showsyntaxerror() |
|
2765 | 2840 | warn('Failure executing file: <%s>' % fname) |
|
2766 | 2841 | except SystemExit,status: |
|
2767 | 2842 | # Code that correctly sets the exit status flag to success (0) |
|
2768 | 2843 | # shouldn't be bothered with a traceback. Note that a plain |
|
2769 | 2844 | # sys.exit() does NOT set the message to 0 (it's empty) so that |
|
2770 | 2845 | # will still get a traceback. Note that the structure of the |
|
2771 | 2846 | # SystemExit exception changed between Python 2.4 and 2.5, so |
|
2772 | 2847 | # the checks must be done in a version-dependent way. |
|
2773 | 2848 | show = False |
|
2774 | 2849 | |
|
2775 | 2850 | if sys.version_info[:2] > (2,5): |
|
2776 | 2851 | if status.message!=0 and not kw['exit_ignore']: |
|
2777 | 2852 | show = True |
|
2778 | 2853 | else: |
|
2779 | 2854 | if status.code and not kw['exit_ignore']: |
|
2780 | 2855 | show = True |
|
2781 | 2856 | if show: |
|
2782 | 2857 | self.showtraceback() |
|
2783 | 2858 | warn('Failure executing file: <%s>' % fname) |
|
2784 | 2859 | except: |
|
2785 | 2860 | self.showtraceback() |
|
2786 | 2861 | warn('Failure executing file: <%s>' % fname) |
|
2787 | 2862 | |
|
2788 | 2863 | syspath_cleanup() |
|
2789 | 2864 | |
|
2790 | 2865 | #************************* end of file <iplib.py> ***************************** |
@@ -1,896 +1,951 b'' | |||
|
1 | 1 | # encoding: utf-8 |
|
2 | 2 | # -*- test-case-name: IPython.kernel.test.test_multiengineclient -*- |
|
3 | 3 | |
|
4 | 4 | """General Classes for IMultiEngine clients.""" |
|
5 | 5 | |
|
6 | 6 | __docformat__ = "restructuredtext en" |
|
7 | 7 | |
|
8 | 8 | #------------------------------------------------------------------------------- |
|
9 | 9 | # Copyright (C) 2008 The IPython Development Team |
|
10 | 10 | # |
|
11 | 11 | # Distributed under the terms of the BSD License. The full license is in |
|
12 | 12 | # the file COPYING, distributed as part of this software. |
|
13 | 13 | #------------------------------------------------------------------------------- |
|
14 | 14 | |
|
15 | 15 | #------------------------------------------------------------------------------- |
|
16 | 16 | # Imports |
|
17 | 17 | #------------------------------------------------------------------------------- |
|
18 | 18 | |
|
19 | 19 | import sys |
|
20 | 20 | import cPickle as pickle |
|
21 | 21 | from types import FunctionType |
|
22 | 22 | import linecache |
|
23 | 23 | |
|
24 | 24 | from twisted.internet import reactor |
|
25 | 25 | from twisted.python import components, log |
|
26 | 26 | from twisted.python.failure import Failure |
|
27 | 27 | from zope.interface import Interface, implements, Attribute |
|
28 | 28 | |
|
29 | 29 | from IPython.ColorANSI import TermColors |
|
30 | 30 | |
|
31 | 31 | from IPython.kernel.twistedutil import blockingCallFromThread |
|
32 | 32 | from IPython.kernel import error |
|
33 | 33 | from IPython.kernel.parallelfunction import ParallelFunction |
|
34 | 34 | from IPython.kernel.mapper import ( |
|
35 | 35 | MultiEngineMapper, |
|
36 | 36 | IMultiEngineMapperFactory, |
|
37 | 37 | IMapper |
|
38 | 38 | ) |
|
39 | 39 | from IPython.kernel import map as Map |
|
40 | 40 | from IPython.kernel import multiengine as me |
|
41 | 41 | from IPython.kernel.multiengine import (IFullMultiEngine, |
|
42 | 42 | IFullSynchronousMultiEngine) |
|
43 | 43 | |
|
44 | 44 | |
|
45 | 45 | #------------------------------------------------------------------------------- |
|
46 | 46 | # Pending Result things |
|
47 | 47 | #------------------------------------------------------------------------------- |
|
48 | 48 | |
|
49 | 49 | class IPendingResult(Interface): |
|
50 | 50 | """A representation of a result that is pending. |
|
51 | 51 | |
|
52 | 52 | This class is similar to Twisted's `Deferred` object, but is designed to be |
|
53 | 53 | used in a synchronous context. |
|
54 | 54 | """ |
|
55 | 55 | |
|
56 | 56 | result_id=Attribute("ID of the deferred on the other side") |
|
57 | 57 | client=Attribute("A client that I came from") |
|
58 | 58 | r=Attribute("An attribute that is a property that calls and returns get_result") |
|
59 | 59 | |
|
60 | 60 | def get_result(default=None, block=True): |
|
61 | 61 | """ |
|
62 | 62 | Get a result that is pending. |
|
63 | 63 | |
|
64 | 64 | :Parameters: |
|
65 | 65 | default |
|
66 | 66 | The value to return if the result is not ready. |
|
67 | 67 | block : boolean |
|
68 | 68 | Should I block for the result. |
|
69 | 69 | |
|
70 | 70 | :Returns: The actual result or the default value. |
|
71 | 71 | """ |
|
72 | 72 | |
|
73 | 73 | def add_callback(f, *args, **kwargs): |
|
74 | 74 | """ |
|
75 | 75 | Add a callback that is called with the result. |
|
76 | 76 | |
|
77 | 77 | If the original result is foo, adding a callback will cause |
|
78 | 78 | f(foo, *args, **kwargs) to be returned instead. If multiple |
|
79 | 79 | callbacks are registered, they are chained together: the result of |
|
80 | 80 | one is passed to the next and so on. |
|
81 | 81 | |
|
82 | 82 | Unlike Twisted's Deferred object, there is no errback chain. Thus |
|
83 | 83 | any exception raised will not be caught and handled. User must |
|
84 | 84 | catch these by hand when calling `get_result`. |
|
85 | 85 | """ |
|
86 | 86 | |
|
87 | 87 | |
|
88 | 88 | class PendingResult(object): |
|
89 | 89 | """A representation of a result that is not yet ready. |
|
90 | 90 | |
|
91 | 91 | A user should not create a `PendingResult` instance by hand. |
|
92 | 92 | |
|
93 | 93 | Methods |
|
94 | 94 | ======= |
|
95 | 95 | |
|
96 | 96 | * `get_result` |
|
97 | 97 | * `add_callback` |
|
98 | 98 | |
|
99 | 99 | Properties |
|
100 | 100 | ========== |
|
101 | 101 | * `r` |
|
102 | 102 | """ |
|
103 | 103 | |
|
104 | 104 | def __init__(self, client, result_id): |
|
105 | 105 | """Create a PendingResult with a result_id and a client instance. |
|
106 | 106 | |
|
107 | 107 | The client should implement `_getPendingResult(result_id, block)`. |
|
108 | 108 | """ |
|
109 | 109 | self.client = client |
|
110 | 110 | self.result_id = result_id |
|
111 | 111 | self.called = False |
|
112 | 112 | self.raised = False |
|
113 | 113 | self.callbacks = [] |
|
114 | 114 | |
|
115 | 115 | def get_result(self, default=None, block=True): |
|
116 | 116 | """Get a result that is pending. |
|
117 | 117 | |
|
118 | 118 | This method will connect to an IMultiEngine adapted controller |
|
119 | 119 | and see if the result is ready. If the action triggers an exception |
|
120 | 120 | raise it and record it. This method records the result/exception once it is |
|
121 | 121 | retrieved. Calling `get_result` again will get this cached result or will |
|
122 | 122 | re-raise the exception. The .r attribute is a property that calls |
|
123 | 123 | `get_result` with block=True. |
|
124 | 124 | |
|
125 | 125 | :Parameters: |
|
126 | 126 | default |
|
127 | 127 | The value to return if the result is not ready. |
|
128 | 128 | block : boolean |
|
129 | 129 | Should I block for the result. |
|
130 | 130 | |
|
131 | 131 | :Returns: The actual result or the default value. |
|
132 | 132 | """ |
|
133 | 133 | |
|
134 | 134 | if self.called: |
|
135 | 135 | if self.raised: |
|
136 | 136 | raise self.result[0], self.result[1], self.result[2] |
|
137 | 137 | else: |
|
138 | 138 | return self.result |
|
139 | 139 | try: |
|
140 | 140 | result = self.client.get_pending_deferred(self.result_id, block) |
|
141 | 141 | except error.ResultNotCompleted: |
|
142 | 142 | return default |
|
143 | 143 | except: |
|
144 | 144 | # Reraise other error, but first record them so they can be reraised |
|
145 | 145 | # later if .r or get_result is called again. |
|
146 | 146 | self.result = sys.exc_info() |
|
147 | 147 | self.called = True |
|
148 | 148 | self.raised = True |
|
149 | 149 | raise |
|
150 | 150 | else: |
|
151 | 151 | for cb in self.callbacks: |
|
152 | 152 | result = cb[0](result, *cb[1], **cb[2]) |
|
153 | 153 | self.result = result |
|
154 | 154 | self.called = True |
|
155 | 155 | return result |
|
156 | 156 | |
|
157 | 157 | def add_callback(self, f, *args, **kwargs): |
|
158 | 158 | """Add a callback that is called with the result. |
|
159 | 159 | |
|
160 | 160 | If the original result is result, adding a callback will cause |
|
161 | 161 | f(result, *args, **kwargs) to be returned instead. If multiple |
|
162 | 162 | callbacks are registered, they are chained together: the result of |
|
163 | 163 | one is passed to the next and so on. |
|
164 | 164 | |
|
165 | 165 | Unlike Twisted's Deferred object, there is no errback chain. Thus |
|
166 | 166 | any exception raised will not be caught and handled. User must |
|
167 | 167 | catch these by hand when calling `get_result`. |
|
168 | 168 | """ |
|
169 | 169 | assert callable(f) |
|
170 | 170 | self.callbacks.append((f, args, kwargs)) |
|
171 | 171 | |
|
172 | 172 | def __cmp__(self, other): |
|
173 | 173 | if self.result_id < other.result_id: |
|
174 | 174 | return -1 |
|
175 | 175 | else: |
|
176 | 176 | return 1 |
|
177 | 177 | |
|
178 | 178 | def _get_r(self): |
|
179 | 179 | return self.get_result(block=True) |
|
180 | 180 | |
|
181 | 181 | r = property(_get_r) |
|
182 | 182 | """This property is a shortcut to a `get_result(block=True)`.""" |
|
183 | 183 | |
|
184 | 184 | |
|
185 | 185 | #------------------------------------------------------------------------------- |
|
186 | 186 | # Pretty printing wrappers for certain lists |
|
187 | 187 | #------------------------------------------------------------------------------- |
|
188 | 188 | |
|
189 | 189 | class ResultList(list): |
|
190 | 190 | """A subclass of list that pretty prints the output of `execute`/`get_result`.""" |
|
191 | 191 | |
|
192 | 192 | def __repr__(self): |
|
193 | 193 | output = [] |
|
194 | 194 | # These colored prompts were not working on Windows |
|
195 | 195 | if sys.platform == 'win32': |
|
196 | 196 | blue = normal = red = green = '' |
|
197 | 197 | else: |
|
198 | 198 | blue = TermColors.Blue |
|
199 | 199 | normal = TermColors.Normal |
|
200 | 200 | red = TermColors.Red |
|
201 | 201 | green = TermColors.Green |
|
202 | 202 | output.append("<Results List>\n") |
|
203 | 203 | for cmd in self: |
|
204 | 204 | if isinstance(cmd, Failure): |
|
205 | 205 | output.append(cmd) |
|
206 | 206 | else: |
|
207 | 207 | target = cmd.get('id',None) |
|
208 | 208 | cmd_num = cmd.get('number',None) |
|
209 | 209 | cmd_stdin = cmd.get('input',{}).get('translated','No Input') |
|
210 | 210 | cmd_stdout = cmd.get('stdout', None) |
|
211 | 211 | cmd_stderr = cmd.get('stderr', None) |
|
212 | 212 | output.append("%s[%i]%s In [%i]:%s %s\n" % \ |
|
213 | 213 | (green, target, |
|
214 | 214 | blue, cmd_num, normal, cmd_stdin)) |
|
215 | 215 | if cmd_stdout: |
|
216 | 216 | output.append("%s[%i]%s Out[%i]:%s %s\n" % \ |
|
217 | 217 | (green, target, |
|
218 | 218 | red, cmd_num, normal, cmd_stdout)) |
|
219 | 219 | if cmd_stderr: |
|
220 | 220 | output.append("%s[%i]%s Err[%i]:\n%s %s" % \ |
|
221 | 221 | (green, target, |
|
222 | 222 | red, cmd_num, normal, cmd_stderr)) |
|
223 | 223 | return ''.join(output) |
|
224 | 224 | |
|
225 | 225 | |
|
226 | 226 | def wrapResultList(result): |
|
227 | 227 | """A function that wraps the output of `execute`/`get_result` -> `ResultList`.""" |
|
228 | 228 | if len(result) == 0: |
|
229 | 229 | result = [result] |
|
230 | 230 | return ResultList(result) |
|
231 | 231 | |
|
232 | 232 | |
|
233 | 233 | class QueueStatusList(list): |
|
234 | 234 | """A subclass of list that pretty prints the output of `queue_status`.""" |
|
235 | 235 | |
|
236 | 236 | def __repr__(self): |
|
237 | 237 | output = [] |
|
238 | 238 | output.append("<Queue Status List>\n") |
|
239 | 239 | for e in self: |
|
240 | 240 | output.append("Engine: %s\n" % repr(e[0])) |
|
241 | 241 | output.append(" Pending: %s\n" % repr(e[1]['pending'])) |
|
242 | 242 | for q in e[1]['queue']: |
|
243 | 243 | output.append(" Command: %s\n" % repr(q)) |
|
244 | 244 | return ''.join(output) |
|
245 | 245 | |
|
246 | 246 | |
|
247 | 247 | #------------------------------------------------------------------------------- |
|
248 | 248 | # InteractiveMultiEngineClient |
|
249 | 249 | #------------------------------------------------------------------------------- |
|
250 | 250 | |
|
251 | 251 | class InteractiveMultiEngineClient(object): |
|
252 | 252 | """A mixin class that add a few methods to a multiengine client. |
|
253 | 253 | |
|
254 | 254 | The methods in this mixin class are designed for interactive usage. |
|
255 | 255 | """ |
|
256 | 256 | |
|
257 | 257 | def activate(self): |
|
258 | 258 | """Make this `MultiEngineClient` active for parallel magic commands. |
|
259 | 259 | |
|
260 | 260 | IPython has a magic command syntax to work with `MultiEngineClient` objects. |
|
261 | 261 | In a given IPython session there is a single active one. While |
|
262 | 262 | there can be many `MultiEngineClient` created and used by the user, |
|
263 | 263 | there is only one active one. The active `MultiEngineClient` is used whenever |
|
264 | 264 | the magic commands %px and %autopx are used. |
|
265 | 265 | |
|
266 | 266 | The activate() method is called on a given `MultiEngineClient` to make it |
|
267 | 267 | active. Once this has been done, the magic commands can be used. |
|
268 | 268 | """ |
|
269 | 269 | |
|
270 | 270 | try: |
|
271 | 271 | __IPYTHON__.activeController = self |
|
272 | 272 | except NameError: |
|
273 | 273 | print "The IPython Controller magics only work within IPython." |
|
274 | 274 | |
|
275 | 275 | def __setitem__(self, key, value): |
|
276 | 276 | """Add a dictionary interface for pushing/pulling. |
|
277 | 277 | |
|
278 | 278 | This functions as a shorthand for `push`. |
|
279 | 279 | |
|
280 | 280 | :Parameters: |
|
281 | 281 | key : str |
|
282 | 282 | What to call the remote object. |
|
283 | 283 | value : object |
|
284 | 284 | The local Python object to push. |
|
285 | 285 | """ |
|
286 | 286 | targets, block = self._findTargetsAndBlock() |
|
287 | 287 | return self.push({key:value}, targets=targets, block=block) |
|
288 | 288 | |
|
289 | 289 | def __getitem__(self, key): |
|
290 | 290 | """Add a dictionary interface for pushing/pulling. |
|
291 | 291 | |
|
292 | 292 | This functions as a shorthand to `pull`. |
|
293 | 293 | |
|
294 | 294 | :Parameters: |
|
295 | 295 | - `key`: A string representing the key. |
|
296 | 296 | """ |
|
297 | 297 | if isinstance(key, str): |
|
298 | 298 | targets, block = self._findTargetsAndBlock() |
|
299 | 299 | return self.pull(key, targets=targets, block=block) |
|
300 | 300 | else: |
|
301 | 301 | raise TypeError("__getitem__ only takes strs") |
|
302 | 302 | |
|
303 | 303 | def __len__(self): |
|
304 | 304 | """Return the number of available engines.""" |
|
305 | 305 | return len(self.get_ids()) |
|
306 | 306 | |
|
307 | 307 | #--------------------------------------------------------------------------- |
|
308 | 308 | # Make this a context manager for with |
|
309 | 309 | #--------------------------------------------------------------------------- |
|
310 | 310 | |
|
311 | 311 | def findsource_file(self,f): |
|
312 | 312 | linecache.checkcache() |
|
313 | 313 | s = findsource(f.f_code) |
|
314 | 314 | lnum = f.f_lineno |
|
315 | 315 | wsource = s[0][f.f_lineno:] |
|
316 | 316 | return strip_whitespace(wsource) |
|
317 | 317 | |
|
318 | 318 | def findsource_ipython(self,f): |
|
319 | 319 | from IPython import ipapi |
|
320 | 320 | self.ip = ipapi.get() |
|
321 | 321 | wsource = [l+'\n' for l in |
|
322 | 322 | self.ip.IP.input_hist_raw[-1].splitlines()[1:]] |
|
323 | 323 | return strip_whitespace(wsource) |
|
324 | 324 | |
|
325 | 325 | def __enter__(self): |
|
326 | 326 | f = sys._getframe(1) |
|
327 | 327 | local_ns = f.f_locals |
|
328 | 328 | global_ns = f.f_globals |
|
329 | 329 | if f.f_code.co_filename == '<ipython console>': |
|
330 | 330 | s = self.findsource_ipython(f) |
|
331 | 331 | else: |
|
332 | 332 | s = self.findsource_file(f) |
|
333 | 333 | |
|
334 | 334 | self._with_context_result = self.execute(s) |
|
335 | 335 | |
|
336 | 336 | def __exit__ (self, etype, value, tb): |
|
337 | 337 | if issubclass(etype,error.StopLocalExecution): |
|
338 | 338 | return True |
|
339 | 339 | |
|
340 | 340 | |
|
341 | 341 | def remote(): |
|
342 | 342 | m = 'Special exception to stop local execution of parallel code.' |
|
343 | 343 | raise error.StopLocalExecution(m) |
|
344 | 344 | |
|
345 | 345 | def strip_whitespace(source): |
|
346 | 346 | # Expand tabs to avoid any confusion. |
|
347 | 347 | wsource = [l.expandtabs(4) for l in source] |
|
348 | 348 | # Detect the indentation level |
|
349 | 349 | done = False |
|
350 | 350 | for line in wsource: |
|
351 | 351 | if line.isspace(): |
|
352 | 352 | continue |
|
353 | 353 | for col,char in enumerate(line): |
|
354 | 354 | if char != ' ': |
|
355 | 355 | done = True |
|
356 | 356 | break |
|
357 | 357 | if done: |
|
358 | 358 | break |
|
359 | 359 | # Now we know how much leading space there is in the code. Next, we |
|
360 | 360 | # extract up to the first line that has less indentation. |
|
361 | 361 | # WARNINGS: we skip comments that may be misindented, but we do NOT yet |
|
362 | 362 | # detect triple quoted strings that may have flush left text. |
|
363 | 363 | for lno,line in enumerate(wsource): |
|
364 | 364 | lead = line[:col] |
|
365 | 365 | if lead.isspace(): |
|
366 | 366 | continue |
|
367 | 367 | else: |
|
368 | 368 | if not lead.lstrip().startswith('#'): |
|
369 | 369 | break |
|
370 | 370 | # The real 'with' source is up to lno |
|
371 | 371 | src_lines = [l[col:] for l in wsource[:lno+1]] |
|
372 | 372 | |
|
373 | 373 | # Finally, check that the source's first non-comment line begins with the |
|
374 | 374 | # special call 'remote()' |
|
375 | 375 | for nline,line in enumerate(src_lines): |
|
376 | 376 | if line.isspace() or line.startswith('#'): |
|
377 | 377 | continue |
|
378 | 378 | if 'remote()' in line: |
|
379 | 379 | break |
|
380 | 380 | else: |
|
381 | 381 | raise ValueError('remote() call missing at the start of code') |
|
382 | 382 | src = ''.join(src_lines[nline+1:]) |
|
383 | 383 | #print 'SRC:\n<<<<<<<>>>>>>>\n%s<<<<<>>>>>>' % src # dbg |
|
384 | 384 | return src |
|
385 | 385 | |
|
386 | 386 | |
|
387 | 387 | #------------------------------------------------------------------------------- |
|
388 | 388 | # The top-level MultiEngine client adaptor |
|
389 | 389 | #------------------------------------------------------------------------------- |
|
390 | 390 | |
|
391 | 391 | |
|
392 | 392 | class IFullBlockingMultiEngineClient(Interface): |
|
393 | 393 | pass |
|
394 | 394 | |
|
395 | 395 | |
|
396 | 396 | class FullBlockingMultiEngineClient(InteractiveMultiEngineClient): |
|
397 | 397 | """ |
|
398 | 398 | A blocking client to the `IMultiEngine` controller interface. |
|
399 | 399 | |
|
400 | 400 | This class allows users to use a set of engines for a parallel |
|
401 | 401 | computation through the `IMultiEngine` interface. In this interface, |
|
402 | 402 | each engine has a specific id (an int) that is used to refer to the |
|
403 | 403 | engine, run code on it, etc. |
|
404 | 404 | """ |
|
405 | 405 | |
|
406 | 406 | implements( |
|
407 | 407 | IFullBlockingMultiEngineClient, |
|
408 | 408 | IMultiEngineMapperFactory, |
|
409 | 409 | IMapper |
|
410 | 410 | ) |
|
411 | 411 | |
|
412 | 412 | def __init__(self, smultiengine): |
|
413 | 413 | self.smultiengine = smultiengine |
|
414 | 414 | self.block = True |
|
415 | 415 | self.targets = 'all' |
|
416 | 416 | |
|
417 | 417 | def _findBlock(self, block=None): |
|
418 | 418 | if block is None: |
|
419 | 419 | return self.block |
|
420 | 420 | else: |
|
421 | 421 | if block in (True, False): |
|
422 | 422 | return block |
|
423 | 423 | else: |
|
424 | 424 | raise ValueError("block must be True or False") |
|
425 | 425 | |
|
426 | 426 | def _findTargets(self, targets=None): |
|
427 | 427 | if targets is None: |
|
428 | 428 | return self.targets |
|
429 | 429 | else: |
|
430 | 430 | if not isinstance(targets, (str,list,tuple,int)): |
|
431 | 431 | raise ValueError("targets must be a str, list, tuple or int") |
|
432 | 432 | return targets |
|
433 | 433 | |
|
434 | 434 | def _findTargetsAndBlock(self, targets=None, block=None): |
|
435 | 435 | return self._findTargets(targets), self._findBlock(block) |
|
436 | 436 | |
|
437 | 437 | def _blockFromThread(self, function, *args, **kwargs): |
|
438 | 438 | block = kwargs.get('block', None) |
|
439 | 439 | if block is None: |
|
440 | 440 | raise error.MissingBlockArgument("'block' keyword argument is missing") |
|
441 | 441 | result = blockingCallFromThread(function, *args, **kwargs) |
|
442 | 442 | if not block: |
|
443 | 443 | result = PendingResult(self, result) |
|
444 | 444 | return result |
|
445 | 445 | |
|
446 | 446 | def get_pending_deferred(self, deferredID, block): |
|
447 | 447 | return blockingCallFromThread(self.smultiengine.get_pending_deferred, deferredID, block) |
|
448 | 448 | |
|
449 | 449 | def barrier(self, pendingResults): |
|
450 | 450 | """Synchronize a set of `PendingResults`. |
|
451 | 451 | |
|
452 | 452 | This method is a synchronization primitive that waits for a set of |
|
453 | 453 | `PendingResult` objects to complete. More specifically, barier does |
|
454 | 454 | the following. |
|
455 | 455 | |
|
456 | 456 | * The `PendingResult`s are sorted by result_id. |
|
457 | 457 | * The `get_result` method is called for each `PendingResult` sequentially |
|
458 | 458 | with block=True. |
|
459 | 459 | * If a `PendingResult` gets a result that is an exception, it is |
|
460 | 460 | trapped and can be re-raised later by calling `get_result` again. |
|
461 | 461 | * The `PendingResult`s are flushed from the controller. |
|
462 | 462 | |
|
463 | 463 | After barrier has been called on a `PendingResult`, its results can |
|
464 | 464 | be retrieved by calling `get_result` again or accesing the `r` attribute |
|
465 | 465 | of the instance. |
|
466 | 466 | """ |
|
467 | 467 | |
|
468 | 468 | # Convert to list for sorting and check class type |
|
469 | 469 | prList = list(pendingResults) |
|
470 | 470 | for pr in prList: |
|
471 | 471 | if not isinstance(pr, PendingResult): |
|
472 | 472 | raise error.NotAPendingResult("Objects passed to barrier must be PendingResult instances") |
|
473 | 473 | |
|
474 | 474 | # Sort the PendingResults so they are in order |
|
475 | 475 | prList.sort() |
|
476 | 476 | # Block on each PendingResult object |
|
477 | 477 | for pr in prList: |
|
478 | 478 | try: |
|
479 | 479 | result = pr.get_result(block=True) |
|
480 | 480 | except Exception: |
|
481 | 481 | pass |
|
482 | 482 | |
|
483 | 483 | def flush(self): |
|
484 | 484 | """ |
|
485 | 485 | Clear all pending deferreds/results from the controller. |
|
486 | 486 | |
|
487 | 487 | For each `PendingResult` that is created by this client, the controller |
|
488 | 488 | holds on to the result for that `PendingResult`. This can be a problem |
|
489 | 489 | if there are a large number of `PendingResult` objects that are created. |
|
490 | 490 | |
|
491 | 491 | Once the result of the `PendingResult` has been retrieved, the result |
|
492 | 492 | is removed from the controller, but if a user doesn't get a result ( |
|
493 | 493 | they just ignore the `PendingResult`) the result is kept forever on the |
|
494 | 494 | controller. This method allows the user to clear out all un-retrieved |
|
495 | 495 | results on the controller. |
|
496 | 496 | """ |
|
497 | 497 | r = blockingCallFromThread(self.smultiengine.clear_pending_deferreds) |
|
498 | 498 | return r |
|
499 | 499 | |
|
500 | 500 | clear_pending_results = flush |
|
501 | 501 | |
|
502 | 502 | #--------------------------------------------------------------------------- |
|
503 | 503 | # IEngineMultiplexer related methods |
|
504 | 504 | #--------------------------------------------------------------------------- |
|
505 | 505 | |
|
506 | 506 | def execute(self, lines, targets=None, block=None): |
|
507 | 507 | """ |
|
508 | 508 | Execute code on a set of engines. |
|
509 | 509 | |
|
510 | 510 | :Parameters: |
|
511 | 511 | lines : str |
|
512 | 512 | The Python code to execute as a string |
|
513 | 513 | targets : id or list of ids |
|
514 | 514 | The engine to use for the execution |
|
515 | 515 | block : boolean |
|
516 | 516 | If False, this method will return the actual result. If False, |
|
517 | 517 | a `PendingResult` is returned which can be used to get the result |
|
518 | 518 | at a later time. |
|
519 | 519 | """ |
|
520 | 520 | targets, block = self._findTargetsAndBlock(targets, block) |
|
521 | 521 | result = blockingCallFromThread(self.smultiengine.execute, lines, |
|
522 | 522 | targets=targets, block=block) |
|
523 | 523 | if block: |
|
524 | 524 | result = ResultList(result) |
|
525 | 525 | else: |
|
526 | 526 | result = PendingResult(self, result) |
|
527 | 527 | result.add_callback(wrapResultList) |
|
528 | 528 | return result |
|
529 | 529 | |
|
530 | 530 | def push(self, namespace, targets=None, block=None): |
|
531 | 531 | """ |
|
532 | 532 | Push a dictionary of keys and values to engines namespace. |
|
533 | 533 | |
|
534 | 534 | Each engine has a persistent namespace. This method is used to push |
|
535 | 535 | Python objects into that namespace. |
|
536 | 536 | |
|
537 | 537 | The objects in the namespace must be pickleable. |
|
538 | 538 | |
|
539 | 539 | :Parameters: |
|
540 | 540 | namespace : dict |
|
541 | 541 | A dict that contains Python objects to be injected into |
|
542 | 542 | the engine persistent namespace. |
|
543 | 543 | targets : id or list of ids |
|
544 | 544 | The engine to use for the execution |
|
545 | 545 | block : boolean |
|
546 | 546 | If False, this method will return the actual result. If False, |
|
547 | 547 | a `PendingResult` is returned which can be used to get the result |
|
548 | 548 | at a later time. |
|
549 | 549 | """ |
|
550 | 550 | targets, block = self._findTargetsAndBlock(targets, block) |
|
551 | 551 | return self._blockFromThread(self.smultiengine.push, namespace, |
|
552 | 552 | targets=targets, block=block) |
|
553 | 553 | |
|
554 | 554 | def pull(self, keys, targets=None, block=None): |
|
555 | 555 | """ |
|
556 | 556 | Pull Python objects by key out of engines namespaces. |
|
557 | 557 | |
|
558 | 558 | :Parameters: |
|
559 | 559 | keys : str or list of str |
|
560 | 560 | The names of the variables to be pulled |
|
561 | 561 | targets : id or list of ids |
|
562 | 562 | The engine to use for the execution |
|
563 | 563 | block : boolean |
|
564 | 564 | If False, this method will return the actual result. If False, |
|
565 | 565 | a `PendingResult` is returned which can be used to get the result |
|
566 | 566 | at a later time. |
|
567 | 567 | """ |
|
568 | 568 | targets, block = self._findTargetsAndBlock(targets, block) |
|
569 | 569 | return self._blockFromThread(self.smultiengine.pull, keys, targets=targets, block=block) |
|
570 | 570 | |
|
571 | 571 | def push_function(self, namespace, targets=None, block=None): |
|
572 | 572 | """ |
|
573 | 573 | Push a Python function to an engine. |
|
574 | 574 | |
|
575 | 575 | This method is used to push a Python function to an engine. This |
|
576 | 576 | method can then be used in code on the engines. Closures are not supported. |
|
577 | 577 | |
|
578 | 578 | :Parameters: |
|
579 | 579 | namespace : dict |
|
580 | 580 | A dict whose values are the functions to be pushed. The keys give |
|
581 | 581 | that names that the function will appear as in the engines |
|
582 | 582 | namespace. |
|
583 | 583 | targets : id or list of ids |
|
584 | 584 | The engine to use for the execution |
|
585 | 585 | block : boolean |
|
586 | 586 | If False, this method will return the actual result. If False, |
|
587 | 587 | a `PendingResult` is returned which can be used to get the result |
|
588 | 588 | at a later time. |
|
589 | 589 | """ |
|
590 | 590 | targets, block = self._findTargetsAndBlock(targets, block) |
|
591 | 591 | return self._blockFromThread(self.smultiengine.push_function, namespace, targets=targets, block=block) |
|
592 | 592 | |
|
593 | 593 | def pull_function(self, keys, targets=None, block=None): |
|
594 | 594 | """ |
|
595 | 595 | Pull a Python function from an engine. |
|
596 | 596 | |
|
597 | 597 | This method is used to pull a Python function from an engine. |
|
598 | 598 | Closures are not supported. |
|
599 | 599 | |
|
600 | 600 | :Parameters: |
|
601 | 601 | keys : str or list of str |
|
602 | 602 | The names of the functions to be pulled |
|
603 | 603 | targets : id or list of ids |
|
604 | 604 | The engine to use for the execution |
|
605 | 605 | block : boolean |
|
606 | 606 | If False, this method will return the actual result. If False, |
|
607 | 607 | a `PendingResult` is returned which can be used to get the result |
|
608 | 608 | at a later time. |
|
609 | 609 | """ |
|
610 | 610 | targets, block = self._findTargetsAndBlock(targets, block) |
|
611 | 611 | return self._blockFromThread(self.smultiengine.pull_function, keys, targets=targets, block=block) |
|
612 | 612 | |
|
613 | 613 | def push_serialized(self, namespace, targets=None, block=None): |
|
614 | 614 | targets, block = self._findTargetsAndBlock(targets, block) |
|
615 | 615 | return self._blockFromThread(self.smultiengine.push_serialized, namespace, targets=targets, block=block) |
|
616 | 616 | |
|
617 | 617 | def pull_serialized(self, keys, targets=None, block=None): |
|
618 | 618 | targets, block = self._findTargetsAndBlock(targets, block) |
|
619 | 619 | return self._blockFromThread(self.smultiengine.pull_serialized, keys, targets=targets, block=block) |
|
620 | 620 | |
|
621 | 621 | def get_result(self, i=None, targets=None, block=None): |
|
622 | 622 | """ |
|
623 | 623 | Get a previous result. |
|
624 | 624 | |
|
625 | 625 | When code is executed in an engine, a dict is created and returned. This |
|
626 | 626 | method retrieves that dict for previous commands. |
|
627 | 627 | |
|
628 | 628 | :Parameters: |
|
629 | 629 | i : int |
|
630 | 630 | The number of the result to get |
|
631 | 631 | targets : id or list of ids |
|
632 | 632 | The engine to use for the execution |
|
633 | 633 | block : boolean |
|
634 | 634 | If False, this method will return the actual result. If False, |
|
635 | 635 | a `PendingResult` is returned which can be used to get the result |
|
636 | 636 | at a later time. |
|
637 | 637 | """ |
|
638 | 638 | targets, block = self._findTargetsAndBlock(targets, block) |
|
639 | 639 | result = blockingCallFromThread(self.smultiengine.get_result, i, targets=targets, block=block) |
|
640 | 640 | if block: |
|
641 | 641 | result = ResultList(result) |
|
642 | 642 | else: |
|
643 | 643 | result = PendingResult(self, result) |
|
644 | 644 | result.add_callback(wrapResultList) |
|
645 | 645 | return result |
|
646 | 646 | |
|
647 | 647 | def reset(self, targets=None, block=None): |
|
648 | 648 | """ |
|
649 | 649 | Reset an engine. |
|
650 | 650 | |
|
651 | 651 | This method clears out the namespace of an engine. |
|
652 | 652 | |
|
653 | 653 | :Parameters: |
|
654 | 654 | targets : id or list of ids |
|
655 | 655 | The engine to use for the execution |
|
656 | 656 | block : boolean |
|
657 | 657 | If False, this method will return the actual result. If False, |
|
658 | 658 | a `PendingResult` is returned which can be used to get the result |
|
659 | 659 | at a later time. |
|
660 | 660 | """ |
|
661 | 661 | targets, block = self._findTargetsAndBlock(targets, block) |
|
662 | 662 | return self._blockFromThread(self.smultiengine.reset, targets=targets, block=block) |
|
663 | 663 | |
|
664 | 664 | def keys(self, targets=None, block=None): |
|
665 | 665 | """ |
|
666 | 666 | Get a list of all the variables in an engine's namespace. |
|
667 | 667 | |
|
668 | 668 | :Parameters: |
|
669 | 669 | targets : id or list of ids |
|
670 | 670 | The engine to use for the execution |
|
671 | 671 | block : boolean |
|
672 | 672 | If False, this method will return the actual result. If False, |
|
673 | 673 | a `PendingResult` is returned which can be used to get the result |
|
674 | 674 | at a later time. |
|
675 | 675 | """ |
|
676 | 676 | targets, block = self._findTargetsAndBlock(targets, block) |
|
677 | 677 | return self._blockFromThread(self.smultiengine.keys, targets=targets, block=block) |
|
678 | 678 | |
|
679 | 679 | def kill(self, controller=False, targets=None, block=None): |
|
680 | 680 | """ |
|
681 | 681 | Kill the engines and controller. |
|
682 | 682 | |
|
683 | 683 | This method is used to stop the engine and controller by calling |
|
684 | 684 | `reactor.stop`. |
|
685 | 685 | |
|
686 | 686 | :Parameters: |
|
687 | 687 | controller : boolean |
|
688 | 688 | If True, kill the engines and controller. If False, just the |
|
689 | 689 | engines |
|
690 | 690 | targets : id or list of ids |
|
691 | 691 | The engine to use for the execution |
|
692 | 692 | block : boolean |
|
693 | 693 | If False, this method will return the actual result. If False, |
|
694 | 694 | a `PendingResult` is returned which can be used to get the result |
|
695 | 695 | at a later time. |
|
696 | 696 | """ |
|
697 | 697 | targets, block = self._findTargetsAndBlock(targets, block) |
|
698 | 698 | return self._blockFromThread(self.smultiengine.kill, controller, targets=targets, block=block) |
|
699 | 699 | |
|
700 | 700 | def clear_queue(self, targets=None, block=None): |
|
701 | 701 | """ |
|
702 | 702 | Clear out the controller's queue for an engine. |
|
703 | 703 | |
|
704 | 704 | The controller maintains a queue for each engine. This clear it out. |
|
705 | 705 | |
|
706 | 706 | :Parameters: |
|
707 | 707 | targets : id or list of ids |
|
708 | 708 | The engine to use for the execution |
|
709 | 709 | block : boolean |
|
710 | 710 | If False, this method will return the actual result. If False, |
|
711 | 711 | a `PendingResult` is returned which can be used to get the result |
|
712 | 712 | at a later time. |
|
713 | 713 | """ |
|
714 | 714 | targets, block = self._findTargetsAndBlock(targets, block) |
|
715 | 715 | return self._blockFromThread(self.smultiengine.clear_queue, targets=targets, block=block) |
|
716 | 716 | |
|
717 | 717 | def queue_status(self, targets=None, block=None): |
|
718 | 718 | """ |
|
719 | 719 | Get the status of an engines queue. |
|
720 | 720 | |
|
721 | 721 | :Parameters: |
|
722 | 722 | targets : id or list of ids |
|
723 | 723 | The engine to use for the execution |
|
724 | 724 | block : boolean |
|
725 | 725 | If False, this method will return the actual result. If False, |
|
726 | 726 | a `PendingResult` is returned which can be used to get the result |
|
727 | 727 | at a later time. |
|
728 | 728 | """ |
|
729 | 729 | targets, block = self._findTargetsAndBlock(targets, block) |
|
730 | 730 | return self._blockFromThread(self.smultiengine.queue_status, targets=targets, block=block) |
|
731 | 731 | |
|
732 | 732 | def set_properties(self, properties, targets=None, block=None): |
|
733 | 733 | targets, block = self._findTargetsAndBlock(targets, block) |
|
734 | 734 | return self._blockFromThread(self.smultiengine.set_properties, properties, targets=targets, block=block) |
|
735 | 735 | |
|
736 | 736 | def get_properties(self, keys=None, targets=None, block=None): |
|
737 | 737 | targets, block = self._findTargetsAndBlock(targets, block) |
|
738 | 738 | return self._blockFromThread(self.smultiengine.get_properties, keys, targets=targets, block=block) |
|
739 | 739 | |
|
740 | 740 | def has_properties(self, keys, targets=None, block=None): |
|
741 | 741 | targets, block = self._findTargetsAndBlock(targets, block) |
|
742 | 742 | return self._blockFromThread(self.smultiengine.has_properties, keys, targets=targets, block=block) |
|
743 | 743 | |
|
744 | 744 | def del_properties(self, keys, targets=None, block=None): |
|
745 | 745 | targets, block = self._findTargetsAndBlock(targets, block) |
|
746 | 746 | return self._blockFromThread(self.smultiengine.del_properties, keys, targets=targets, block=block) |
|
747 | 747 | |
|
748 | 748 | def clear_properties(self, targets=None, block=None): |
|
749 | 749 | targets, block = self._findTargetsAndBlock(targets, block) |
|
750 | 750 | return self._blockFromThread(self.smultiengine.clear_properties, targets=targets, block=block) |
|
751 | 751 | |
|
752 | 752 | #--------------------------------------------------------------------------- |
|
753 | 753 | # IMultiEngine related methods |
|
754 | 754 | #--------------------------------------------------------------------------- |
|
755 | 755 | |
|
756 | 756 | def get_ids(self): |
|
757 | 757 | """ |
|
758 | 758 | Returns the ids of currently registered engines. |
|
759 | 759 | """ |
|
760 | 760 | result = blockingCallFromThread(self.smultiengine.get_ids) |
|
761 | 761 | return result |
|
762 | 762 | |
|
763 | 763 | #--------------------------------------------------------------------------- |
|
764 | 764 | # IMultiEngineCoordinator |
|
765 | 765 | #--------------------------------------------------------------------------- |
|
766 | 766 | |
|
767 | 767 | def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None): |
|
768 | 768 | """ |
|
769 | 769 | Partition a Python sequence and send the partitions to a set of engines. |
|
770 | 770 | """ |
|
771 | 771 | targets, block = self._findTargetsAndBlock(targets, block) |
|
772 | 772 | return self._blockFromThread(self.smultiengine.scatter, key, seq, |
|
773 | 773 | dist, flatten, targets=targets, block=block) |
|
774 | 774 | |
|
775 | 775 | def gather(self, key, dist='b', targets=None, block=None): |
|
776 | 776 | """ |
|
777 | 777 | Gather a partitioned sequence on a set of engines as a single local seq. |
|
778 | 778 | """ |
|
779 | 779 | targets, block = self._findTargetsAndBlock(targets, block) |
|
780 | 780 | return self._blockFromThread(self.smultiengine.gather, key, dist, |
|
781 | 781 | targets=targets, block=block) |
|
782 | 782 | |
|
783 | 783 | def raw_map(self, func, seq, dist='b', targets=None, block=None): |
|
784 | 784 | """ |
|
785 | 785 | A parallelized version of Python's builtin map. |
|
786 | 786 | |
|
787 | 787 | This has a slightly different syntax than the builtin `map`. |
|
788 | 788 | This is needed because we need to have keyword arguments and thus |
|
789 | 789 | can't use *args to capture all the sequences. Instead, they must |
|
790 | 790 | be passed in a list or tuple. |
|
791 | 791 | |
|
792 | 792 | raw_map(func, seqs) -> map(func, seqs[0], seqs[1], ...) |
|
793 | 793 | |
|
794 | 794 | Most users will want to use parallel functions or the `mapper` |
|
795 | 795 | and `map` methods for an API that follows that of the builtin |
|
796 | 796 | `map`. |
|
797 | 797 | """ |
|
798 | 798 | targets, block = self._findTargetsAndBlock(targets, block) |
|
799 | 799 | return self._blockFromThread(self.smultiengine.raw_map, func, seq, |
|
800 | 800 | dist, targets=targets, block=block) |
|
801 | 801 | |
|
802 | 802 | def map(self, func, *sequences): |
|
803 | 803 | """ |
|
804 | 804 | A parallel version of Python's builtin `map` function. |
|
805 | 805 | |
|
806 | 806 | This method applies a function to sequences of arguments. It |
|
807 | 807 | follows the same syntax as the builtin `map`. |
|
808 | 808 | |
|
809 | 809 | This method creates a mapper objects by calling `self.mapper` with |
|
810 | 810 | no arguments and then uses that mapper to do the mapping. See |
|
811 | 811 | the documentation of `mapper` for more details. |
|
812 | 812 | """ |
|
813 | 813 | return self.mapper().map(func, *sequences) |
|
814 | 814 | |
|
815 | 815 | def mapper(self, dist='b', targets='all', block=None): |
|
816 | 816 | """ |
|
817 | 817 | Create a mapper object that has a `map` method. |
|
818 | 818 | |
|
819 | 819 | This method returns an object that implements the `IMapper` |
|
820 | 820 | interface. This method is a factory that is used to control how |
|
821 | 821 | the map happens. |
|
822 | 822 | |
|
823 | 823 | :Parameters: |
|
824 | 824 | dist : str |
|
825 | 825 | What decomposition to use, 'b' is the only one supported |
|
826 | 826 | currently |
|
827 | 827 | targets : str, int, sequence of ints |
|
828 | 828 | Which engines to use for the map |
|
829 | 829 | block : boolean |
|
830 | 830 | Should calls to `map` block or not |
|
831 | 831 | """ |
|
832 | 832 | return MultiEngineMapper(self, dist, targets, block) |
|
833 | 833 | |
|
834 | 834 | def parallel(self, dist='b', targets=None, block=None): |
|
835 | 835 | """ |
|
836 | 836 | A decorator that turns a function into a parallel function. |
|
837 | 837 | |
|
838 | 838 | This can be used as: |
|
839 | 839 | |
|
840 | 840 | @parallel() |
|
841 | 841 | def f(x, y) |
|
842 | 842 | ... |
|
843 | 843 | |
|
844 | 844 | f(range(10), range(10)) |
|
845 | 845 | |
|
846 | 846 | This causes f(0,0), f(1,1), ... to be called in parallel. |
|
847 | 847 | |
|
848 | 848 | :Parameters: |
|
849 | 849 | dist : str |
|
850 | 850 | What decomposition to use, 'b' is the only one supported |
|
851 | 851 | currently |
|
852 | 852 | targets : str, int, sequence of ints |
|
853 | 853 | Which engines to use for the map |
|
854 | 854 | block : boolean |
|
855 | 855 | Should calls to `map` block or not |
|
856 | 856 | """ |
|
857 | 857 | targets, block = self._findTargetsAndBlock(targets, block) |
|
858 | 858 | mapper = self.mapper(dist, targets, block) |
|
859 | 859 | pf = ParallelFunction(mapper) |
|
860 | 860 | return pf |
|
861 | 861 | |
|
862 | 862 | #--------------------------------------------------------------------------- |
|
863 | 863 | # IMultiEngineExtras |
|
864 | 864 | #--------------------------------------------------------------------------- |
|
865 | 865 | |
|
866 | 866 | def zip_pull(self, keys, targets=None, block=None): |
|
867 | 867 | targets, block = self._findTargetsAndBlock(targets, block) |
|
868 | 868 | return self._blockFromThread(self.smultiengine.zip_pull, keys, |
|
869 | 869 | targets=targets, block=block) |
|
870 | 870 | |
|
871 | 871 | def run(self, filename, targets=None, block=None): |
|
872 | 872 | """ |
|
873 | 873 | Run a Python code in a file on the engines. |
|
874 | 874 | |
|
875 | 875 | :Parameters: |
|
876 | 876 | filename : str |
|
877 | 877 | The name of the local file to run |
|
878 | 878 | targets : id or list of ids |
|
879 | 879 | The engine to use for the execution |
|
880 | 880 | block : boolean |
|
881 | 881 | If False, this method will return the actual result. If False, |
|
882 | 882 | a `PendingResult` is returned which can be used to get the result |
|
883 | 883 | at a later time. |
|
884 | 884 | """ |
|
885 | 885 | targets, block = self._findTargetsAndBlock(targets, block) |
|
886 | 886 | return self._blockFromThread(self.smultiengine.run, filename, |
|
887 | 887 | targets=targets, block=block) |
|
888 | ||
|
889 | def benchmark(self, push_size=10000): | |
|
890 | """ | |
|
891 | Run performance benchmarks for the current IPython cluster. | |
|
892 | ||
|
893 | This method tests both the latency of sending command and data to the | |
|
894 | engines as well as the throughput of sending large objects to the | |
|
895 | engines using push. The latency is measured by having one or more | |
|
896 | engines execute the command 'pass'. The throughput is measure by | |
|
897 | sending an NumPy array of size `push_size` to one or more engines. | |
|
898 | ||
|
899 | These benchmarks will vary widely on different hardware and networks | |
|
900 | and thus can be used to get an idea of the performance characteristics | |
|
901 | of a particular configuration of an IPython controller and engines. | |
|
902 | ||
|
903 | This function is not testable within our current testing framework. | |
|
904 | """ | |
|
905 | import timeit, __builtin__ | |
|
906 | __builtin__._mec_self = self | |
|
907 | benchmarks = {} | |
|
908 | repeat = 3 | |
|
909 | count = 10 | |
|
910 | ||
|
911 | timer = timeit.Timer('_mec_self.execute("pass",0)') | |
|
912 | result = 1000*min(timer.repeat(repeat,count))/count | |
|
913 | benchmarks['single_engine_latency'] = (result,'msec') | |
|
914 | ||
|
915 | timer = timeit.Timer('_mec_self.execute("pass")') | |
|
916 | result = 1000*min(timer.repeat(repeat,count))/count | |
|
917 | benchmarks['all_engine_latency'] = (result,'msec') | |
|
888 | 918 | |
|
919 | try: | |
|
920 | import numpy as np | |
|
921 | except: | |
|
922 | pass | |
|
923 | else: | |
|
924 | timer = timeit.Timer( | |
|
925 | "_mec_self.push(d)", | |
|
926 | "import numpy as np; d = dict(a=np.zeros(%r,dtype='float64'))" % push_size | |
|
927 | ) | |
|
928 | result = min(timer.repeat(repeat,count))/count | |
|
929 | benchmarks['all_engine_push'] = (1e-6*push_size*8/result, 'MB/sec') | |
|
930 | ||
|
931 | try: | |
|
932 | import numpy as np | |
|
933 | except: | |
|
934 | pass | |
|
935 | else: | |
|
936 | timer = timeit.Timer( | |
|
937 | "_mec_self.push(d,0)", | |
|
938 | "import numpy as np; d = dict(a=np.zeros(%r,dtype='float64'))" % push_size | |
|
939 | ) | |
|
940 | result = min(timer.repeat(repeat,count))/count | |
|
941 | benchmarks['single_engine_push'] = (1e-6*push_size*8/result, 'MB/sec') | |
|
942 | ||
|
943 | return benchmarks | |
|
889 | 944 | |
|
890 | 945 | |
|
891 | 946 | components.registerAdapter(FullBlockingMultiEngineClient, |
|
892 | 947 | IFullSynchronousMultiEngine, IFullBlockingMultiEngineClient) |
|
893 | 948 | |
|
894 | 949 | |
|
895 | 950 | |
|
896 | 951 |
@@ -1,723 +1,783 b'' | |||
|
1 | 1 | #!/usr/bin/env python |
|
2 | 2 | # encoding: utf-8 |
|
3 | 3 | |
|
4 | 4 | """Start an IPython cluster = (controller + engines).""" |
|
5 | 5 | |
|
6 | 6 | #----------------------------------------------------------------------------- |
|
7 | 7 | # Copyright (C) 2008 The IPython Development Team |
|
8 | 8 | # |
|
9 | 9 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | 10 | # the file COPYING, distributed as part of this software. |
|
11 | 11 | #----------------------------------------------------------------------------- |
|
12 | 12 | |
|
13 | 13 | #----------------------------------------------------------------------------- |
|
14 | 14 | # Imports |
|
15 | 15 | #----------------------------------------------------------------------------- |
|
16 | 16 | |
|
17 | 17 | import os |
|
18 | 18 | import re |
|
19 | 19 | import sys |
|
20 | 20 | import signal |
|
21 | 21 | import tempfile |
|
22 | 22 | pjoin = os.path.join |
|
23 | 23 | |
|
24 | 24 | from twisted.internet import reactor, defer |
|
25 | 25 | from twisted.internet.protocol import ProcessProtocol |
|
26 | 26 | from twisted.internet.error import ProcessDone, ProcessTerminated |
|
27 | 27 | from twisted.internet.utils import getProcessOutput |
|
28 | 28 | from twisted.python import failure, log |
|
29 | 29 | |
|
30 | 30 | from IPython.external import argparse |
|
31 | 31 | from IPython.external import Itpl |
|
32 | 32 | from IPython.genutils import get_ipython_dir, num_cpus |
|
33 | 33 | from IPython.kernel.fcutil import have_crypto |
|
34 | 34 | from IPython.kernel.error import SecurityError |
|
35 | 35 | from IPython.kernel.fcutil import have_crypto |
|
36 | 36 | from IPython.kernel.twistedutil import gatherBoth |
|
37 | 37 | from IPython.kernel.util import printer |
|
38 | 38 | |
|
39 | 39 | |
|
40 | 40 | #----------------------------------------------------------------------------- |
|
41 | 41 | # General process handling code |
|
42 | 42 | #----------------------------------------------------------------------------- |
|
43 | 43 | |
|
44 | 44 | def find_exe(cmd): |
|
45 | 45 | try: |
|
46 | 46 | import win32api |
|
47 | 47 | except ImportError: |
|
48 | 48 | raise ImportError('you need to have pywin32 installed for this to work') |
|
49 | 49 | else: |
|
50 | 50 | try: |
|
51 | 51 | (path, offest) = win32api.SearchPath(os.environ['PATH'],cmd + '.exe') |
|
52 | 52 | except: |
|
53 | 53 | (path, offset) = win32api.SearchPath(os.environ['PATH'],cmd + '.bat') |
|
54 | 54 | return path |
|
55 | 55 | |
|
56 | 56 | class ProcessStateError(Exception): |
|
57 | 57 | pass |
|
58 | 58 | |
|
59 | 59 | class UnknownStatus(Exception): |
|
60 | 60 | pass |
|
61 | 61 | |
|
62 | 62 | class LauncherProcessProtocol(ProcessProtocol): |
|
63 | 63 | """ |
|
64 | 64 | A ProcessProtocol to go with the ProcessLauncher. |
|
65 | 65 | """ |
|
66 | 66 | def __init__(self, process_launcher): |
|
67 | 67 | self.process_launcher = process_launcher |
|
68 | 68 | |
|
69 | 69 | def connectionMade(self): |
|
70 | 70 | self.process_launcher.fire_start_deferred(self.transport.pid) |
|
71 | 71 | |
|
72 | 72 | def processEnded(self, status): |
|
73 | 73 | value = status.value |
|
74 | 74 | if isinstance(value, ProcessDone): |
|
75 | 75 | self.process_launcher.fire_stop_deferred(0) |
|
76 | 76 | elif isinstance(value, ProcessTerminated): |
|
77 | 77 | self.process_launcher.fire_stop_deferred( |
|
78 | 78 | {'exit_code':value.exitCode, |
|
79 | 79 | 'signal':value.signal, |
|
80 | 80 | 'status':value.status |
|
81 | 81 | } |
|
82 | 82 | ) |
|
83 | 83 | else: |
|
84 | 84 | raise UnknownStatus("unknown exit status, this is probably a bug in Twisted") |
|
85 | 85 | |
|
86 | 86 | def outReceived(self, data): |
|
87 | 87 | log.msg(data) |
|
88 | 88 | |
|
89 | 89 | def errReceived(self, data): |
|
90 | 90 | log.err(data) |
|
91 | 91 | |
|
92 | 92 | class ProcessLauncher(object): |
|
93 | 93 | """ |
|
94 | 94 | Start and stop an external process in an asynchronous manner. |
|
95 | 95 | |
|
96 | 96 | Currently this uses deferreds to notify other parties of process state |
|
97 | 97 | changes. This is an awkward design and should be moved to using |
|
98 | 98 | a formal NotificationCenter. |
|
99 | 99 | """ |
|
100 | 100 | def __init__(self, cmd_and_args): |
|
101 | 101 | self.cmd = cmd_and_args[0] |
|
102 | 102 | self.args = cmd_and_args |
|
103 | 103 | self._reset() |
|
104 | 104 | |
|
105 | 105 | def _reset(self): |
|
106 | 106 | self.process_protocol = None |
|
107 | 107 | self.pid = None |
|
108 | 108 | self.start_deferred = None |
|
109 | 109 | self.stop_deferreds = [] |
|
110 | 110 | self.state = 'before' # before, running, or after |
|
111 | 111 | |
|
112 | 112 | @property |
|
113 | 113 | def running(self): |
|
114 | 114 | if self.state == 'running': |
|
115 | 115 | return True |
|
116 | 116 | else: |
|
117 | 117 | return False |
|
118 | 118 | |
|
119 | 119 | def fire_start_deferred(self, pid): |
|
120 | 120 | self.pid = pid |
|
121 | 121 | self.state = 'running' |
|
122 | 122 | log.msg('Process %r has started with pid=%i' % (self.args, pid)) |
|
123 | 123 | self.start_deferred.callback(pid) |
|
124 | 124 | |
|
125 | 125 | def start(self): |
|
126 | 126 | if self.state == 'before': |
|
127 | 127 | self.process_protocol = LauncherProcessProtocol(self) |
|
128 | 128 | self.start_deferred = defer.Deferred() |
|
129 | 129 | self.process_transport = reactor.spawnProcess( |
|
130 | 130 | self.process_protocol, |
|
131 | 131 | self.cmd, |
|
132 | 132 | self.args, |
|
133 | 133 | env=os.environ |
|
134 | 134 | ) |
|
135 | 135 | return self.start_deferred |
|
136 | 136 | else: |
|
137 | 137 | s = 'the process has already been started and has state: %r' % \ |
|
138 | 138 | self.state |
|
139 | 139 | return defer.fail(ProcessStateError(s)) |
|
140 | 140 | |
|
141 | 141 | def get_stop_deferred(self): |
|
142 | 142 | if self.state == 'running' or self.state == 'before': |
|
143 | 143 | d = defer.Deferred() |
|
144 | 144 | self.stop_deferreds.append(d) |
|
145 | 145 | return d |
|
146 | 146 | else: |
|
147 | 147 | s = 'this process is already complete' |
|
148 | 148 | return defer.fail(ProcessStateError(s)) |
|
149 | 149 | |
|
150 | 150 | def fire_stop_deferred(self, exit_code): |
|
151 | 151 | log.msg('Process %r has stopped with %r' % (self.args, exit_code)) |
|
152 | 152 | self.state = 'after' |
|
153 | 153 | for d in self.stop_deferreds: |
|
154 | 154 | d.callback(exit_code) |
|
155 | 155 | |
|
156 | 156 | def signal(self, sig): |
|
157 | 157 | """ |
|
158 | 158 | Send a signal to the process. |
|
159 | 159 | |
|
160 | 160 | The argument sig can be ('KILL','INT', etc.) or any signal number. |
|
161 | 161 | """ |
|
162 | 162 | if self.state == 'running': |
|
163 | 163 | self.process_transport.signalProcess(sig) |
|
164 | 164 | |
|
165 | 165 | # def __del__(self): |
|
166 | 166 | # self.signal('KILL') |
|
167 | 167 | |
|
168 | 168 | def interrupt_then_kill(self, delay=1.0): |
|
169 | 169 | self.signal('INT') |
|
170 | 170 | reactor.callLater(delay, self.signal, 'KILL') |
|
171 | 171 | |
|
172 | 172 | |
|
173 | 173 | #----------------------------------------------------------------------------- |
|
174 | 174 | # Code for launching controller and engines |
|
175 | 175 | #----------------------------------------------------------------------------- |
|
176 | 176 | |
|
177 | 177 | |
|
178 | 178 | class ControllerLauncher(ProcessLauncher): |
|
179 | 179 | |
|
180 | 180 | def __init__(self, extra_args=None): |
|
181 | 181 | if sys.platform == 'win32': |
|
182 | 182 | # This logic is needed because the ipcontroller script doesn't |
|
183 | 183 | # always get installed in the same way or in the same location. |
|
184 | 184 | from IPython.kernel.scripts import ipcontroller |
|
185 | 185 | script_location = ipcontroller.__file__.replace('.pyc', '.py') |
|
186 | 186 | # The -u option here turns on unbuffered output, which is required |
|
187 | 187 | # on Win32 to prevent wierd conflict and problems with Twisted |
|
188 | 188 | args = [find_exe('python'), '-u', script_location] |
|
189 | 189 | else: |
|
190 | 190 | args = ['ipcontroller'] |
|
191 | 191 | self.extra_args = extra_args |
|
192 | 192 | if extra_args is not None: |
|
193 | 193 | args.extend(extra_args) |
|
194 | 194 | |
|
195 | 195 | ProcessLauncher.__init__(self, args) |
|
196 | 196 | |
|
197 | 197 | |
|
198 | 198 | class EngineLauncher(ProcessLauncher): |
|
199 | 199 | |
|
200 | 200 | def __init__(self, extra_args=None): |
|
201 | 201 | if sys.platform == 'win32': |
|
202 | 202 | # This logic is needed because the ipcontroller script doesn't |
|
203 | 203 | # always get installed in the same way or in the same location. |
|
204 | 204 | from IPython.kernel.scripts import ipengine |
|
205 | 205 | script_location = ipengine.__file__.replace('.pyc', '.py') |
|
206 | 206 | # The -u option here turns on unbuffered output, which is required |
|
207 | 207 | # on Win32 to prevent wierd conflict and problems with Twisted |
|
208 | 208 | args = [find_exe('python'), '-u', script_location] |
|
209 | 209 | else: |
|
210 | 210 | args = ['ipengine'] |
|
211 | 211 | self.extra_args = extra_args |
|
212 | 212 | if extra_args is not None: |
|
213 | 213 | args.extend(extra_args) |
|
214 | 214 | |
|
215 | 215 | ProcessLauncher.__init__(self, args) |
|
216 | 216 | |
|
217 | 217 | |
|
218 | 218 | class LocalEngineSet(object): |
|
219 | 219 | |
|
220 | 220 | def __init__(self, extra_args=None): |
|
221 | 221 | self.extra_args = extra_args |
|
222 | 222 | self.launchers = [] |
|
223 | 223 | |
|
224 | 224 | def start(self, n): |
|
225 | 225 | dlist = [] |
|
226 | 226 | for i in range(n): |
|
227 | 227 | el = EngineLauncher(extra_args=self.extra_args) |
|
228 | 228 | d = el.start() |
|
229 | 229 | self.launchers.append(el) |
|
230 | 230 | dlist.append(d) |
|
231 | 231 | dfinal = gatherBoth(dlist, consumeErrors=True) |
|
232 | 232 | dfinal.addCallback(self._handle_start) |
|
233 | 233 | return dfinal |
|
234 | 234 | |
|
235 | 235 | def _handle_start(self, r): |
|
236 | 236 | log.msg('Engines started with pids: %r' % r) |
|
237 | 237 | return r |
|
238 | 238 | |
|
239 | 239 | def _handle_stop(self, r): |
|
240 | 240 | log.msg('Engines received signal: %r' % r) |
|
241 | 241 | return r |
|
242 | 242 | |
|
243 | 243 | def signal(self, sig): |
|
244 | 244 | dlist = [] |
|
245 | 245 | for el in self.launchers: |
|
246 | 246 | d = el.get_stop_deferred() |
|
247 | 247 | dlist.append(d) |
|
248 | 248 | el.signal(sig) |
|
249 | 249 | dfinal = gatherBoth(dlist, consumeErrors=True) |
|
250 | 250 | dfinal.addCallback(self._handle_stop) |
|
251 | 251 | return dfinal |
|
252 | 252 | |
|
253 | 253 | def interrupt_then_kill(self, delay=1.0): |
|
254 | 254 | dlist = [] |
|
255 | 255 | for el in self.launchers: |
|
256 | 256 | d = el.get_stop_deferred() |
|
257 | 257 | dlist.append(d) |
|
258 | 258 | el.interrupt_then_kill(delay) |
|
259 | 259 | dfinal = gatherBoth(dlist, consumeErrors=True) |
|
260 | 260 | dfinal.addCallback(self._handle_stop) |
|
261 | 261 | return dfinal |
|
262 | 262 | |
|
263 | 263 | |
|
264 | 264 | class BatchEngineSet(object): |
|
265 | 265 | |
|
266 | 266 | # Subclasses must fill these in. See PBSEngineSet |
|
267 | 267 | submit_command = '' |
|
268 | 268 | delete_command = '' |
|
269 | 269 | job_id_regexp = '' |
|
270 | 270 | |
|
271 | 271 | def __init__(self, template_file, **kwargs): |
|
272 | 272 | self.template_file = template_file |
|
273 | 273 | self.context = {} |
|
274 | 274 | self.context.update(kwargs) |
|
275 | 275 | self.batch_file = self.template_file+'-run' |
|
276 | 276 | |
|
277 | 277 | def parse_job_id(self, output): |
|
278 | 278 | m = re.match(self.job_id_regexp, output) |
|
279 | 279 | if m is not None: |
|
280 | 280 | job_id = m.group() |
|
281 | 281 | else: |
|
282 | 282 | raise Exception("job id couldn't be determined: %s" % output) |
|
283 | 283 | self.job_id = job_id |
|
284 | 284 | log.msg('Job started with job id: %r' % job_id) |
|
285 | 285 | return job_id |
|
286 | 286 | |
|
287 | 287 | def write_batch_script(self, n): |
|
288 | 288 | self.context['n'] = n |
|
289 | 289 | template = open(self.template_file, 'r').read() |
|
290 | 290 | log.msg('Using template for batch script: %s' % self.template_file) |
|
291 | 291 | script_as_string = Itpl.itplns(template, self.context) |
|
292 | 292 | log.msg('Writing instantiated batch script: %s' % self.batch_file) |
|
293 | 293 | f = open(self.batch_file,'w') |
|
294 | 294 | f.write(script_as_string) |
|
295 | 295 | f.close() |
|
296 | 296 | |
|
297 | 297 | def handle_error(self, f): |
|
298 | 298 | f.printTraceback() |
|
299 | 299 | f.raiseException() |
|
300 | 300 | |
|
301 | 301 | def start(self, n): |
|
302 | 302 | self.write_batch_script(n) |
|
303 | 303 | d = getProcessOutput(self.submit_command, |
|
304 | 304 | [self.batch_file],env=os.environ) |
|
305 | 305 | d.addCallback(self.parse_job_id) |
|
306 | 306 | d.addErrback(self.handle_error) |
|
307 | 307 | return d |
|
308 | 308 | |
|
309 | 309 | def kill(self): |
|
310 | 310 | d = getProcessOutput(self.delete_command, |
|
311 | 311 | [self.job_id],env=os.environ) |
|
312 | 312 | return d |
|
313 | 313 | |
|
314 | 314 | class PBSEngineSet(BatchEngineSet): |
|
315 | 315 | |
|
316 | 316 | submit_command = 'qsub' |
|
317 | 317 | delete_command = 'qdel' |
|
318 | 318 | job_id_regexp = '\d+' |
|
319 | 319 | |
|
320 | 320 | def __init__(self, template_file, **kwargs): |
|
321 | 321 | BatchEngineSet.__init__(self, template_file, **kwargs) |
|
322 | 322 | |
|
323 | 323 | |
|
324 | 324 | sshx_template="""#!/bin/sh |
|
325 | 325 | "$@" &> /dev/null & |
|
326 | 326 | echo $! |
|
327 | 327 | """ |
|
328 | 328 | |
|
329 | 329 | engine_killer_template="""#!/bin/sh |
|
330 | 330 | ps -fu `whoami` | grep '[i]pengine' | awk '{print $2}' | xargs kill -TERM |
|
331 | 331 | """ |
|
332 | 332 | |
|
333 | 333 | class SSHEngineSet(object): |
|
334 | 334 | sshx_template=sshx_template |
|
335 | 335 | engine_killer_template=engine_killer_template |
|
336 | 336 | |
|
337 | 337 | def __init__(self, engine_hosts, sshx=None, ipengine="ipengine"): |
|
338 | 338 | """Start a controller on localhost and engines using ssh. |
|
339 | 339 | |
|
340 | 340 | The engine_hosts argument is a dict with hostnames as keys and |
|
341 | 341 | the number of engine (int) as values. sshx is the name of a local |
|
342 | 342 | file that will be used to run remote commands. This file is used |
|
343 | 343 | to setup the environment properly. |
|
344 | 344 | """ |
|
345 | 345 | |
|
346 | 346 | self.temp_dir = tempfile.gettempdir() |
|
347 | 347 | if sshx is not None: |
|
348 | 348 | self.sshx = sshx |
|
349 | 349 | else: |
|
350 | 350 | # Write the sshx.sh file locally from our template. |
|
351 | 351 | self.sshx = os.path.join( |
|
352 | 352 | self.temp_dir, |
|
353 | 353 | '%s-main-sshx.sh' % os.environ['USER'] |
|
354 | 354 | ) |
|
355 | 355 | f = open(self.sshx, 'w') |
|
356 | 356 | f.writelines(self.sshx_template) |
|
357 | 357 | f.close() |
|
358 | 358 | self.engine_command = ipengine |
|
359 | 359 | self.engine_hosts = engine_hosts |
|
360 | 360 | # Write the engine killer script file locally from our template. |
|
361 | 361 | self.engine_killer = os.path.join( |
|
362 | 362 | self.temp_dir, |
|
363 | 363 | '%s-local-engine_killer.sh' % os.environ['USER'] |
|
364 | 364 | ) |
|
365 | 365 | f = open(self.engine_killer, 'w') |
|
366 | 366 | f.writelines(self.engine_killer_template) |
|
367 | 367 | f.close() |
|
368 | 368 | |
|
369 | 369 | def start(self, send_furl=False): |
|
370 | 370 | dlist = [] |
|
371 | 371 | for host in self.engine_hosts.keys(): |
|
372 | 372 | count = self.engine_hosts[host] |
|
373 | 373 | d = self._start(host, count, send_furl) |
|
374 | 374 | dlist.append(d) |
|
375 | 375 | return gatherBoth(dlist, consumeErrors=True) |
|
376 | 376 | |
|
377 | 377 | def _start(self, hostname, count=1, send_furl=False): |
|
378 | 378 | if send_furl: |
|
379 | 379 | d = self._scp_furl(hostname) |
|
380 | 380 | else: |
|
381 | 381 | d = defer.succeed(None) |
|
382 | 382 | d.addCallback(lambda r: self._scp_sshx(hostname)) |
|
383 | 383 | d.addCallback(lambda r: self._ssh_engine(hostname, count)) |
|
384 | 384 | return d |
|
385 | 385 | |
|
386 | 386 | def _scp_furl(self, hostname): |
|
387 | 387 | scp_cmd = "scp ~/.ipython/security/ipcontroller-engine.furl %s:.ipython/security/" % (hostname) |
|
388 | 388 | cmd_list = scp_cmd.split() |
|
389 | 389 | cmd_list[1] = os.path.expanduser(cmd_list[1]) |
|
390 | 390 | log.msg('Copying furl file: %s' % scp_cmd) |
|
391 | 391 | d = getProcessOutput(cmd_list[0], cmd_list[1:], env=os.environ) |
|
392 | 392 | return d |
|
393 | 393 | |
|
394 | 394 | def _scp_sshx(self, hostname): |
|
395 | 395 | scp_cmd = "scp %s %s:%s/%s-sshx.sh" % ( |
|
396 | 396 | self.sshx, hostname, |
|
397 | 397 | self.temp_dir, os.environ['USER'] |
|
398 | 398 | ) |
|
399 | 399 | |
|
400 | 400 | log.msg("Copying sshx: %s" % scp_cmd) |
|
401 | 401 | sshx_scp = scp_cmd.split() |
|
402 | 402 | d = getProcessOutput(sshx_scp[0], sshx_scp[1:], env=os.environ) |
|
403 | 403 | return d |
|
404 | 404 | |
|
405 | 405 | def _ssh_engine(self, hostname, count): |
|
406 | 406 | exec_engine = "ssh %s sh %s/%s-sshx.sh %s" % ( |
|
407 | 407 | hostname, self.temp_dir, |
|
408 | 408 | os.environ['USER'], self.engine_command |
|
409 | 409 | ) |
|
410 | 410 | cmds = exec_engine.split() |
|
411 | 411 | dlist = [] |
|
412 | 412 | log.msg("about to start engines...") |
|
413 | 413 | for i in range(count): |
|
414 | 414 | log.msg('Starting engines: %s' % exec_engine) |
|
415 | 415 | d = getProcessOutput(cmds[0], cmds[1:], env=os.environ) |
|
416 | 416 | dlist.append(d) |
|
417 | 417 | return gatherBoth(dlist, consumeErrors=True) |
|
418 | 418 | |
|
419 | 419 | def kill(self): |
|
420 | 420 | dlist = [] |
|
421 | 421 | for host in self.engine_hosts.keys(): |
|
422 | 422 | d = self._killall(host) |
|
423 | 423 | dlist.append(d) |
|
424 | 424 | return gatherBoth(dlist, consumeErrors=True) |
|
425 | 425 | |
|
426 | 426 | def _killall(self, hostname): |
|
427 | 427 | d = self._scp_engine_killer(hostname) |
|
428 | 428 | d.addCallback(lambda r: self._ssh_kill(hostname)) |
|
429 | 429 | # d.addErrback(self._exec_err) |
|
430 | 430 | return d |
|
431 | 431 | |
|
432 | 432 | def _scp_engine_killer(self, hostname): |
|
433 | 433 | scp_cmd = "scp %s %s:%s/%s-engine_killer.sh" % ( |
|
434 | 434 | self.engine_killer, |
|
435 | 435 | hostname, |
|
436 | 436 | self.temp_dir, |
|
437 | 437 | os.environ['USER'] |
|
438 | 438 | ) |
|
439 | 439 | cmds = scp_cmd.split() |
|
440 | 440 | log.msg('Copying engine_killer: %s' % scp_cmd) |
|
441 | 441 | d = getProcessOutput(cmds[0], cmds[1:], env=os.environ) |
|
442 | 442 | return d |
|
443 | 443 | |
|
444 | 444 | def _ssh_kill(self, hostname): |
|
445 | 445 | kill_cmd = "ssh %s sh %s/%s-engine_killer.sh" % ( |
|
446 | 446 | hostname, |
|
447 | 447 | self.temp_dir, |
|
448 | 448 | os.environ['USER'] |
|
449 | 449 | ) |
|
450 | 450 | log.msg('Killing engine: %s' % kill_cmd) |
|
451 | 451 | kill_cmd = kill_cmd.split() |
|
452 | 452 | d = getProcessOutput(kill_cmd[0], kill_cmd[1:], env=os.environ) |
|
453 | 453 | return d |
|
454 | 454 | |
|
455 | 455 | def _exec_err(self, r): |
|
456 | 456 | log.msg(r) |
|
457 | 457 | |
|
458 | 458 | #----------------------------------------------------------------------------- |
|
459 | 459 | # Main functions for the different types of clusters |
|
460 | 460 | #----------------------------------------------------------------------------- |
|
461 | 461 | |
|
462 | 462 | # TODO: |
|
463 | 463 | # The logic in these codes should be moved into classes like LocalCluster |
|
464 | 464 | # MpirunCluster, PBSCluster, etc. This would remove alot of the duplications. |
|
465 | 465 | # The main functions should then just parse the command line arguments, create |
|
466 | 466 | # the appropriate class and call a 'start' method. |
|
467 | 467 | |
|
468 | 468 | def check_security(args, cont_args): |
|
469 | 469 | if (not args.x or not args.y) and not have_crypto: |
|
470 | 470 | log.err(""" |
|
471 | 471 | OpenSSL/pyOpenSSL is not available, so we can't run in secure mode. |
|
472 | 472 | Try running ipcluster with the -xy flags: ipcluster local -xy -n 4""") |
|
473 | 473 | reactor.stop() |
|
474 | 474 | return False |
|
475 | 475 | if args.x: |
|
476 | 476 | cont_args.append('-x') |
|
477 | 477 | if args.y: |
|
478 | 478 | cont_args.append('-y') |
|
479 | 479 | return True |
|
480 | 480 | |
|
481 | def check_reuse(args, cont_args): | |
|
482 | if args.r: | |
|
483 | cont_args.append('-r') | |
|
484 | if args.client_port == 0 or args.engine_port == 0: | |
|
485 | log.err(""" | |
|
486 | To reuse FURL files, you must also set the client and engine ports using | |
|
487 | the --client-port and --engine-port options.""") | |
|
488 | reactor.stop() | |
|
489 | return False | |
|
490 | cont_args.append('--client-port=%i' % args.client_port) | |
|
491 | cont_args.append('--engine-port=%i' % args.engine_port) | |
|
492 | return True | |
|
481 | 493 | |
|
482 | 494 | def main_local(args): |
|
483 | 495 | cont_args = [] |
|
484 | 496 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) |
|
485 | ||
|
497 | ||
|
486 | 498 | # Check security settings before proceeding |
|
487 | 499 | if not check_security(args, cont_args): |
|
488 | 500 | return |
|
489 | ||
|
501 | ||
|
502 | # See if we are reusing FURL files | |
|
503 | if not check_reuse(args, cont_args): | |
|
504 | return | |
|
505 | ||
|
490 | 506 | cl = ControllerLauncher(extra_args=cont_args) |
|
491 | 507 | dstart = cl.start() |
|
492 | 508 | def start_engines(cont_pid): |
|
493 | 509 | engine_args = [] |
|
494 | 510 | engine_args.append('--logfile=%s' % \ |
|
495 | 511 | pjoin(args.logdir,'ipengine%s-' % cont_pid)) |
|
496 | 512 | eset = LocalEngineSet(extra_args=engine_args) |
|
497 | 513 | def shutdown(signum, frame): |
|
498 | 514 | log.msg('Stopping local cluster') |
|
499 | 515 | # We are still playing with the times here, but these seem |
|
500 | 516 | # to be reliable in allowing everything to exit cleanly. |
|
501 | 517 | eset.interrupt_then_kill(0.5) |
|
502 | 518 | cl.interrupt_then_kill(0.5) |
|
503 | 519 | reactor.callLater(1.0, reactor.stop) |
|
504 | 520 | signal.signal(signal.SIGINT,shutdown) |
|
505 | 521 | d = eset.start(args.n) |
|
506 | 522 | return d |
|
507 | 523 | def delay_start(cont_pid): |
|
508 | 524 | # This is needed because the controller doesn't start listening |
|
509 | 525 | # right when it starts and the controller needs to write |
|
510 | 526 | # furl files for the engine to pick up |
|
511 | 527 | reactor.callLater(1.0, start_engines, cont_pid) |
|
512 | 528 | dstart.addCallback(delay_start) |
|
513 | 529 | dstart.addErrback(lambda f: f.raiseException()) |
|
514 | 530 | |
|
515 | 531 | |
|
516 |
def main_mpi |
|
|
532 | def main_mpi(args): | |
|
517 | 533 | cont_args = [] |
|
518 | 534 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) |
|
519 | ||
|
535 | ||
|
520 | 536 | # Check security settings before proceeding |
|
521 | 537 | if not check_security(args, cont_args): |
|
522 | 538 | return |
|
523 | ||
|
539 | ||
|
540 | # See if we are reusing FURL files | |
|
541 | if not check_reuse(args, cont_args): | |
|
542 | return | |
|
543 | ||
|
524 | 544 | cl = ControllerLauncher(extra_args=cont_args) |
|
525 | 545 | dstart = cl.start() |
|
526 | 546 | def start_engines(cont_pid): |
|
527 |
raw_args = [ |
|
|
547 | raw_args = [args.cmd] | |
|
528 | 548 | raw_args.extend(['-n',str(args.n)]) |
|
529 | 549 | raw_args.append('ipengine') |
|
530 | 550 | raw_args.append('-l') |
|
531 | 551 | raw_args.append(pjoin(args.logdir,'ipengine%s-' % cont_pid)) |
|
532 | 552 | if args.mpi: |
|
533 | 553 | raw_args.append('--mpi=%s' % args.mpi) |
|
534 | 554 | eset = ProcessLauncher(raw_args) |
|
535 | 555 | def shutdown(signum, frame): |
|
536 | 556 | log.msg('Stopping local cluster') |
|
537 | 557 | # We are still playing with the times here, but these seem |
|
538 | 558 | # to be reliable in allowing everything to exit cleanly. |
|
539 | 559 | eset.interrupt_then_kill(1.0) |
|
540 | 560 | cl.interrupt_then_kill(1.0) |
|
541 | 561 | reactor.callLater(2.0, reactor.stop) |
|
542 | 562 | signal.signal(signal.SIGINT,shutdown) |
|
543 | 563 | d = eset.start() |
|
544 | 564 | return d |
|
545 | 565 | def delay_start(cont_pid): |
|
546 | 566 | # This is needed because the controller doesn't start listening |
|
547 | 567 | # right when it starts and the controller needs to write |
|
548 | 568 | # furl files for the engine to pick up |
|
549 | 569 | reactor.callLater(1.0, start_engines, cont_pid) |
|
550 | 570 | dstart.addCallback(delay_start) |
|
551 | 571 | dstart.addErrback(lambda f: f.raiseException()) |
|
552 | 572 | |
|
553 | 573 | |
|
554 | 574 | def main_pbs(args): |
|
555 | 575 | cont_args = [] |
|
556 | 576 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) |
|
557 | ||
|
577 | ||
|
558 | 578 | # Check security settings before proceeding |
|
559 | 579 | if not check_security(args, cont_args): |
|
560 | 580 | return |
|
561 | ||
|
581 | ||
|
582 | # See if we are reusing FURL files | |
|
583 | if not check_reuse(args, cont_args): | |
|
584 | return | |
|
585 | ||
|
562 | 586 | cl = ControllerLauncher(extra_args=cont_args) |
|
563 | 587 | dstart = cl.start() |
|
564 | 588 | def start_engines(r): |
|
565 | 589 | pbs_set = PBSEngineSet(args.pbsscript) |
|
566 | 590 | def shutdown(signum, frame): |
|
567 | 591 | log.msg('Stopping pbs cluster') |
|
568 | 592 | d = pbs_set.kill() |
|
569 | 593 | d.addBoth(lambda _: cl.interrupt_then_kill(1.0)) |
|
570 | 594 | d.addBoth(lambda _: reactor.callLater(2.0, reactor.stop)) |
|
571 | 595 | signal.signal(signal.SIGINT,shutdown) |
|
572 | 596 | d = pbs_set.start(args.n) |
|
573 | 597 | return d |
|
574 | 598 | dstart.addCallback(start_engines) |
|
575 | 599 | dstart.addErrback(lambda f: f.raiseException()) |
|
576 | 600 | |
|
577 | 601 | |
|
578 | 602 | def main_ssh(args): |
|
579 | 603 | """Start a controller on localhost and engines using ssh. |
|
580 | 604 | |
|
581 | 605 | Your clusterfile should look like:: |
|
582 | 606 | |
|
583 | 607 | send_furl = False # True, if you want |
|
584 | 608 | engines = { |
|
585 | 609 | 'engine_host1' : engine_count, |
|
586 | 610 | 'engine_host2' : engine_count2 |
|
587 | 611 | } |
|
588 | 612 | """ |
|
589 | 613 | clusterfile = {} |
|
590 | 614 | execfile(args.clusterfile, clusterfile) |
|
591 | 615 | if not clusterfile.has_key('send_furl'): |
|
592 | 616 | clusterfile['send_furl'] = False |
|
593 | 617 | |
|
594 | 618 | cont_args = [] |
|
595 | 619 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) |
|
596 | 620 | |
|
597 | 621 | # Check security settings before proceeding |
|
598 | 622 | if not check_security(args, cont_args): |
|
599 | 623 | return |
|
600 | 624 | |
|
625 | # See if we are reusing FURL files | |
|
626 | if not check_reuse(args, cont_args): | |
|
627 | return | |
|
628 | ||
|
601 | 629 | cl = ControllerLauncher(extra_args=cont_args) |
|
602 | 630 | dstart = cl.start() |
|
603 | 631 | def start_engines(cont_pid): |
|
604 | 632 | ssh_set = SSHEngineSet(clusterfile['engines'], sshx=args.sshx) |
|
605 | 633 | def shutdown(signum, frame): |
|
606 | 634 | d = ssh_set.kill() |
|
607 | # d.addErrback(log.err) | |
|
608 | 635 | cl.interrupt_then_kill(1.0) |
|
609 | 636 | reactor.callLater(2.0, reactor.stop) |
|
610 | 637 | signal.signal(signal.SIGINT,shutdown) |
|
611 | 638 | d = ssh_set.start(clusterfile['send_furl']) |
|
612 | 639 | return d |
|
613 | 640 | |
|
614 | 641 | def delay_start(cont_pid): |
|
615 | 642 | reactor.callLater(1.0, start_engines, cont_pid) |
|
616 | 643 | |
|
617 | 644 | dstart.addCallback(delay_start) |
|
618 | 645 | dstart.addErrback(lambda f: f.raiseException()) |
|
619 | 646 | |
|
620 | 647 | |
|
621 | 648 | def get_args(): |
|
622 | 649 | base_parser = argparse.ArgumentParser(add_help=False) |
|
623 | 650 | base_parser.add_argument( |
|
651 | '-r', | |
|
652 | action='store_true', | |
|
653 | dest='r', | |
|
654 | help='try to reuse FURL files. Use with --client-port and --engine-port' | |
|
655 | ) | |
|
656 | base_parser.add_argument( | |
|
657 | '--client-port', | |
|
658 | type=int, | |
|
659 | dest='client_port', | |
|
660 | help='the port the controller will listen on for client connections', | |
|
661 | default=0 | |
|
662 | ) | |
|
663 | base_parser.add_argument( | |
|
664 | '--engine-port', | |
|
665 | type=int, | |
|
666 | dest='engine_port', | |
|
667 | help='the port the controller will listen on for engine connections', | |
|
668 | default=0 | |
|
669 | ) | |
|
670 | base_parser.add_argument( | |
|
624 | 671 | '-x', |
|
625 | 672 | action='store_true', |
|
626 | 673 | dest='x', |
|
627 | 674 | help='turn off client security' |
|
628 | 675 | ) |
|
629 | 676 | base_parser.add_argument( |
|
630 | 677 | '-y', |
|
631 | 678 | action='store_true', |
|
632 | 679 | dest='y', |
|
633 | 680 | help='turn off engine security' |
|
634 | 681 | ) |
|
635 | 682 | base_parser.add_argument( |
|
636 | 683 | "--logdir", |
|
637 | 684 | type=str, |
|
638 | 685 | dest="logdir", |
|
639 | 686 | help="directory to put log files (default=$IPYTHONDIR/log)", |
|
640 | 687 | default=pjoin(get_ipython_dir(),'log') |
|
641 | 688 | ) |
|
642 | 689 | base_parser.add_argument( |
|
643 | 690 | "-n", |
|
644 | 691 | "--num", |
|
645 | 692 | type=int, |
|
646 | 693 | dest="n", |
|
647 | 694 | default=2, |
|
648 | 695 | help="the number of engines to start" |
|
649 | 696 | ) |
|
650 | 697 | |
|
651 | 698 | parser = argparse.ArgumentParser( |
|
652 | 699 | description='IPython cluster startup. This starts a controller and\ |
|
653 | 700 | engines using various approaches. THIS IS A TECHNOLOGY PREVIEW AND\ |
|
654 | 701 | THE API WILL CHANGE SIGNIFICANTLY BEFORE THE FINAL RELEASE.' |
|
655 | 702 | ) |
|
656 | 703 | subparsers = parser.add_subparsers( |
|
657 | 704 | help='available cluster types. For help, do "ipcluster TYPE --help"') |
|
658 | 705 | |
|
659 | 706 | parser_local = subparsers.add_parser( |
|
660 | 707 | 'local', |
|
661 | 708 | help='run a local cluster', |
|
662 | 709 | parents=[base_parser] |
|
663 | 710 | ) |
|
664 | 711 | parser_local.set_defaults(func=main_local) |
|
665 | 712 | |
|
666 | 713 | parser_mpirun = subparsers.add_parser( |
|
667 | 714 | 'mpirun', |
|
668 | help='run a cluster using mpirun', | |
|
715 | help='run a cluster using mpirun (mpiexec also works)', | |
|
669 | 716 | parents=[base_parser] |
|
670 | 717 | ) |
|
671 | 718 | parser_mpirun.add_argument( |
|
672 | 719 | "--mpi", |
|
673 | 720 | type=str, |
|
674 | 721 | dest="mpi", # Don't put a default here to allow no MPI support |
|
675 | 722 | help="how to call MPI_Init (default=mpi4py)" |
|
676 | 723 | ) |
|
677 | parser_mpirun.set_defaults(func=main_mpirun) | |
|
724 | parser_mpirun.set_defaults(func=main_mpi, cmd='mpirun') | |
|
725 | ||
|
726 | parser_mpiexec = subparsers.add_parser( | |
|
727 | 'mpiexec', | |
|
728 | help='run a cluster using mpiexec (mpirun also works)', | |
|
729 | parents=[base_parser] | |
|
730 | ) | |
|
731 | parser_mpiexec.add_argument( | |
|
732 | "--mpi", | |
|
733 | type=str, | |
|
734 | dest="mpi", # Don't put a default here to allow no MPI support | |
|
735 | help="how to call MPI_Init (default=mpi4py)" | |
|
736 | ) | |
|
737 | parser_mpiexec.set_defaults(func=main_mpi, cmd='mpiexec') | |
|
678 | 738 | |
|
679 | 739 | parser_pbs = subparsers.add_parser( |
|
680 | 740 | 'pbs', |
|
681 | 741 | help='run a pbs cluster', |
|
682 | 742 | parents=[base_parser] |
|
683 | 743 | ) |
|
684 | 744 | parser_pbs.add_argument( |
|
685 | 745 | '--pbs-script', |
|
686 | 746 | type=str, |
|
687 | 747 | dest='pbsscript', |
|
688 | 748 | help='PBS script template', |
|
689 | 749 | default='pbs.template' |
|
690 | 750 | ) |
|
691 | 751 | parser_pbs.set_defaults(func=main_pbs) |
|
692 | 752 | |
|
693 | 753 | parser_ssh = subparsers.add_parser( |
|
694 | 754 | 'ssh', |
|
695 | 755 | help='run a cluster using ssh, should have ssh-keys setup', |
|
696 | 756 | parents=[base_parser] |
|
697 | 757 | ) |
|
698 | 758 | parser_ssh.add_argument( |
|
699 | 759 | '--clusterfile', |
|
700 | 760 | type=str, |
|
701 | 761 | dest='clusterfile', |
|
702 | 762 | help='python file describing the cluster', |
|
703 | 763 | default='clusterfile.py', |
|
704 | 764 | ) |
|
705 | 765 | parser_ssh.add_argument( |
|
706 | 766 | '--sshx', |
|
707 | 767 | type=str, |
|
708 | 768 | dest='sshx', |
|
709 | 769 | help='sshx launcher helper' |
|
710 | 770 | ) |
|
711 | 771 | parser_ssh.set_defaults(func=main_ssh) |
|
712 | 772 | |
|
713 | 773 | args = parser.parse_args() |
|
714 | 774 | return args |
|
715 | 775 | |
|
716 | 776 | def main(): |
|
717 | 777 | args = get_args() |
|
718 | 778 | reactor.callWhenRunning(args.func, args) |
|
719 | 779 | log.startLogging(sys.stdout) |
|
720 | 780 | reactor.run() |
|
721 | 781 | |
|
722 | 782 | if __name__ == '__main__': |
|
723 | 783 | main() |
@@ -1,98 +1,100 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """IPython Test Suite Runner. |
|
3 | 3 | |
|
4 | 4 | This module provides a main entry point to a user script to test IPython itself |
|
5 | 5 | from the command line. The main() routine can be used in a similar manner to |
|
6 | 6 | the ``nosetests`` script, and it takes similar arguments, but if no arguments |
|
7 | 7 | are given it defaults to testing all of IPython. This should be preferred to |
|
8 | 8 | using plain ``nosetests`` because a number of nose plugins necessary to test |
|
9 | 9 | IPython correctly are automatically configured by this code. |
|
10 | 10 | """ |
|
11 | 11 | |
|
12 | 12 | #----------------------------------------------------------------------------- |
|
13 | 13 | # Module imports |
|
14 | 14 | #----------------------------------------------------------------------------- |
|
15 | 15 | |
|
16 | 16 | # stdlib |
|
17 | 17 | import sys |
|
18 | 18 | import warnings |
|
19 | 19 | |
|
20 | 20 | # third-party |
|
21 | 21 | import nose.plugins.builtin |
|
22 | 22 | from nose.core import TestProgram |
|
23 | 23 | |
|
24 | 24 | # Our own imports |
|
25 | 25 | from IPython.testing.plugin.ipdoctest import IPythonDoctest |
|
26 | 26 | |
|
27 | 27 | #----------------------------------------------------------------------------- |
|
28 | 28 | # Constants and globals |
|
29 | 29 | #----------------------------------------------------------------------------- |
|
30 | 30 | |
|
31 | 31 | # For the IPythonDoctest plugin, we need to exclude certain patterns that cause |
|
32 | 32 | # testing problems. We should strive to minimize the number of skipped |
|
33 | 33 | # modules, since this means untested code. As the testing machinery |
|
34 | 34 | # solidifies, this list should eventually become empty. |
|
35 | 35 | EXCLUDE = ['IPython/external/', |
|
36 | 36 | 'IPython/platutils_win32', |
|
37 | 37 | 'IPython/frontend/cocoa', |
|
38 | 38 | 'IPython_doctest_plugin', |
|
39 | 39 | 'IPython/Gnuplot', |
|
40 | 40 | 'IPython/Extensions/ipy_', |
|
41 | 41 | 'IPython/Extensions/clearcmd', |
|
42 | 42 | 'IPython/Extensions/PhysicalQIn', |
|
43 | 43 | 'IPython/Extensions/scitedirector', |
|
44 | 44 | ] |
|
45 | 45 | |
|
46 | 46 | #----------------------------------------------------------------------------- |
|
47 | 47 | # Functions and classes |
|
48 | 48 | #----------------------------------------------------------------------------- |
|
49 | 49 | |
|
50 | 50 | def main(): |
|
51 | 51 | """Run the IPython test suite. |
|
52 | 52 | """ |
|
53 | 53 | |
|
54 | 54 | warnings.filterwarnings('ignore', |
|
55 | 55 | 'This will be removed soon. Use IPython.testing.util instead') |
|
56 | 56 | |
|
57 | 57 | argv = sys.argv + [ |
|
58 | 58 | # Loading ipdoctest causes problems with Twisted. |
|
59 | 59 | # I am removing this as a temporary fix to get the |
|
60 | 60 | # test suite back into working shape. Our nose |
|
61 | 61 | # plugin needs to be gone through with a fine |
|
62 | 62 | # toothed comb to find what is causing the problem. |
|
63 | 63 | # '--with-ipdoctest', |
|
64 | '--doctest-tests','--doctest-extension=txt', | |
|
64 | '--ipdoctest-tests','--ipdoctest-extension=txt', | |
|
65 | 65 | '--detailed-errors', |
|
66 | 66 | |
|
67 | 67 | # We add --exe because of setuptools' imbecility (it |
|
68 | 68 | # blindly does chmod +x on ALL files). Nose does the |
|
69 | 69 | # right thing and it tries to avoid executables, |
|
70 | 70 | # setuptools unfortunately forces our hand here. This |
|
71 | 71 | # has been discussed on the distutils list and the |
|
72 | 72 | # setuptools devs refuse to fix this problem! |
|
73 | 73 | '--exe', |
|
74 | 74 | ] |
|
75 | 75 | |
|
76 | 76 | # Detect if any tests were required by explicitly calling an IPython |
|
77 | 77 | # submodule or giving a specific path |
|
78 | 78 | has_tests = False |
|
79 | 79 | for arg in sys.argv: |
|
80 | 80 | if 'IPython' in arg or arg.endswith('.py') or \ |
|
81 | 81 | (':' in arg and '.py' in arg): |
|
82 | 82 | has_tests = True |
|
83 | 83 | break |
|
84 | ||
|
84 | 85 | # If nothing was specifically requested, test full IPython |
|
85 | 86 | if not has_tests: |
|
86 | 87 | argv.append('IPython') |
|
87 | 88 | |
|
88 |
# Construct list of plugins, omitting the existing doctest plugin |
|
|
89 | # Construct list of plugins, omitting the existing doctest plugin, which | |
|
90 | # ours replaces (and extends). | |
|
89 | 91 | plugins = [IPythonDoctest(EXCLUDE)] |
|
90 | 92 | for p in nose.plugins.builtin.plugins: |
|
91 | 93 | plug = p() |
|
92 | 94 | if plug.name == 'doctest': |
|
93 | 95 | continue |
|
94 | 96 | |
|
95 | 97 | #print '*** adding plugin:',plug.name # dbg |
|
96 | 98 | plugins.append(plug) |
|
97 | 99 | |
|
98 | 100 | TestProgram(argv=argv,plugins=plugins) |
@@ -1,850 +1,875 b'' | |||
|
1 | 1 | """Nose Plugin that supports IPython doctests. |
|
2 | 2 | |
|
3 | 3 | Limitations: |
|
4 | 4 | |
|
5 | 5 | - When generating examples for use as doctests, make sure that you have |
|
6 | 6 | pretty-printing OFF. This can be done either by starting ipython with the |
|
7 | 7 | flag '--nopprint', by setting pprint to 0 in your ipythonrc file, or by |
|
8 | 8 | interactively disabling it with %Pprint. This is required so that IPython |
|
9 | 9 | output matches that of normal Python, which is used by doctest for internal |
|
10 | 10 | execution. |
|
11 | 11 | |
|
12 | 12 | - Do not rely on specific prompt numbers for results (such as using |
|
13 | 13 | '_34==True', for example). For IPython tests run via an external process the |
|
14 | 14 | prompt numbers may be different, and IPython tests run as normal python code |
|
15 | 15 | won't even have these special _NN variables set at all. |
|
16 | 16 | """ |
|
17 | 17 | |
|
18 | ||
|
19 | 18 | #----------------------------------------------------------------------------- |
|
20 | 19 | # Module imports |
|
21 | 20 | |
|
22 | 21 | # From the standard library |
|
23 | 22 | import __builtin__ |
|
24 | 23 | import commands |
|
25 | 24 | import doctest |
|
26 | 25 | import inspect |
|
27 | 26 | import logging |
|
28 | 27 | import os |
|
29 | 28 | import re |
|
30 | 29 | import sys |
|
31 | 30 | import traceback |
|
32 | 31 | import unittest |
|
33 | 32 | |
|
34 | 33 | from inspect import getmodule |
|
35 | 34 | from StringIO import StringIO |
|
36 | 35 | |
|
37 | 36 | # We are overriding the default doctest runner, so we need to import a few |
|
38 | 37 | # things from doctest directly |
|
39 | 38 | from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE, |
|
40 | 39 | _unittest_reportflags, DocTestRunner, |
|
41 | 40 | _extract_future_flags, pdb, _OutputRedirectingPdb, |
|
42 | 41 | _exception_traceback, |
|
43 | 42 | linecache) |
|
44 | 43 | |
|
45 | 44 | # Third-party modules |
|
46 | 45 | import nose.core |
|
47 | 46 | |
|
48 | 47 | from nose.plugins import doctests, Plugin |
|
49 | 48 | from nose.util import anyp, getpackage, test_address, resolve_name, tolist |
|
50 | 49 | |
|
51 | 50 | #----------------------------------------------------------------------------- |
|
52 | 51 | # Module globals and other constants |
|
53 | 52 | |
|
54 | 53 | log = logging.getLogger(__name__) |
|
55 | 54 | |
|
56 | 55 | ########################################################################### |
|
57 | 56 | # *** HACK *** |
|
58 | 57 | # We must start our own ipython object and heavily muck with it so that all the |
|
59 | 58 | # modifications IPython makes to system behavior don't send the doctest |
|
60 | 59 | # machinery into a fit. This code should be considered a gross hack, but it |
|
61 | 60 | # gets the job done. |
|
62 | 61 | |
|
63 | 62 | |
|
64 | 63 | # Hack to modify the %run command so we can sync the user's namespace with the |
|
65 | 64 | # test globals. Once we move over to a clean magic system, this will be done |
|
66 | 65 | # with much less ugliness. |
|
67 | 66 | |
|
68 | 67 | class py_file_finder(object): |
|
69 | 68 | def __init__(self,test_filename): |
|
70 | 69 | self.test_filename = test_filename |
|
71 | 70 | |
|
72 | 71 | def __call__(self,name): |
|
73 | 72 | from IPython.genutils import get_py_filename |
|
74 | 73 | try: |
|
75 | 74 | return get_py_filename(name) |
|
76 | 75 | except IOError: |
|
77 | 76 | test_dir = os.path.dirname(self.test_filename) |
|
78 | 77 | new_path = os.path.join(test_dir,name) |
|
79 | 78 | return get_py_filename(new_path) |
|
80 | 79 | |
|
81 | 80 | |
|
82 | 81 | def _run_ns_sync(self,arg_s,runner=None): |
|
83 | 82 | """Modified version of %run that syncs testing namespaces. |
|
84 | 83 | |
|
85 | 84 | This is strictly needed for running doctests that call %run. |
|
86 | 85 | """ |
|
87 | 86 | |
|
88 | 87 | finder = py_file_finder(_run_ns_sync.test_filename) |
|
89 | 88 | out = _ip.IP.magic_run_ori(arg_s,runner,finder) |
|
90 | 89 | _run_ns_sync.test_globs.update(_ip.user_ns) |
|
91 | 90 | return out |
|
92 | 91 | |
|
93 | 92 | |
|
94 | 93 | class ipnsdict(dict): |
|
95 | 94 | """A special subclass of dict for use as an IPython namespace in doctests. |
|
96 | 95 | |
|
97 | 96 | This subclass adds a simple checkpointing capability so that when testing |
|
98 | 97 | machinery clears it (we use it as the test execution context), it doesn't |
|
99 | 98 | get completely destroyed. |
|
100 | 99 | """ |
|
101 | 100 | |
|
102 | 101 | def __init__(self,*a): |
|
103 | 102 | dict.__init__(self,*a) |
|
104 | 103 | self._savedict = {} |
|
105 | 104 | |
|
106 | 105 | def clear(self): |
|
107 | 106 | dict.clear(self) |
|
108 | 107 | self.update(self._savedict) |
|
109 | 108 | |
|
110 | 109 | def _checkpoint(self): |
|
111 | 110 | self._savedict.clear() |
|
112 | 111 | self._savedict.update(self) |
|
113 | 112 | |
|
114 | 113 | def update(self,other): |
|
115 | 114 | self._checkpoint() |
|
116 | 115 | dict.update(self,other) |
|
117 | 116 | # If '_' is in the namespace, python won't set it when executing code, |
|
118 | 117 | # and we have examples that test it. So we ensure that the namespace |
|
119 | 118 | # is always 'clean' of it before it's used for test code execution. |
|
120 | 119 | self.pop('_',None) |
|
121 | 120 | |
|
122 | 121 | |
|
123 | 122 | def start_ipython(): |
|
124 | 123 | """Start a global IPython shell, which we need for IPython-specific syntax. |
|
125 | 124 | """ |
|
125 | ||
|
126 | # This function should only ever run once! | |
|
127 | if hasattr(start_ipython,'already_called'): | |
|
128 | return | |
|
129 | start_ipython.already_called = True | |
|
130 | ||
|
131 | # Ok, first time we're called, go ahead | |
|
126 | 132 | import new |
|
127 | 133 | |
|
128 | 134 | import IPython |
|
129 | 135 | |
|
130 | 136 | def xsys(cmd): |
|
131 | 137 | """Execute a command and print its output. |
|
132 | 138 | |
|
133 | 139 | This is just a convenience function to replace the IPython system call |
|
134 | 140 | with one that is more doctest-friendly. |
|
135 | 141 | """ |
|
136 | 142 | cmd = _ip.IP.var_expand(cmd,depth=1) |
|
137 | 143 | sys.stdout.write(commands.getoutput(cmd)) |
|
138 | 144 | sys.stdout.flush() |
|
139 | 145 | |
|
140 | 146 | # Store certain global objects that IPython modifies |
|
141 | 147 | _displayhook = sys.displayhook |
|
142 | 148 | _excepthook = sys.excepthook |
|
143 | 149 | _main = sys.modules.get('__main__') |
|
144 | 150 | |
|
145 | 151 | # Start IPython instance. We customize it to start with minimal frills. |
|
146 | 152 | user_ns,global_ns = IPython.ipapi.make_user_namespaces(ipnsdict(),dict()) |
|
147 | 153 | IPython.Shell.IPShell(['--colors=NoColor','--noterm_title'], |
|
148 | 154 | user_ns,global_ns) |
|
149 | 155 | |
|
150 | 156 | # Deactivate the various python system hooks added by ipython for |
|
151 | 157 | # interactive convenience so we don't confuse the doctest system |
|
152 | 158 | sys.modules['__main__'] = _main |
|
153 | 159 | sys.displayhook = _displayhook |
|
154 | 160 | sys.excepthook = _excepthook |
|
155 | 161 | |
|
156 | 162 | # So that ipython magics and aliases can be doctested (they work by making |
|
157 | 163 | # a call into a global _ip object) |
|
158 | 164 | _ip = IPython.ipapi.get() |
|
159 | 165 | __builtin__._ip = _ip |
|
160 | 166 | |
|
161 | 167 | # Modify the IPython system call with one that uses getoutput, so that we |
|
162 | 168 | # can capture subcommands and print them to Python's stdout, otherwise the |
|
163 | 169 | # doctest machinery would miss them. |
|
164 | 170 | _ip.system = xsys |
|
165 | 171 | |
|
166 | 172 | # Also patch our %run function in. |
|
167 | 173 | im = new.instancemethod(_run_ns_sync,_ip.IP, _ip.IP.__class__) |
|
168 | 174 | _ip.IP.magic_run_ori = _ip.IP.magic_run |
|
169 | 175 | _ip.IP.magic_run = im |
|
170 | 176 | |
|
171 | 177 | # The start call MUST be made here. I'm not sure yet why it doesn't work if |
|
172 | 178 | # it is made later, at plugin initialization time, but in all my tests, that's |
|
173 | 179 | # the case. |
|
174 | 180 | start_ipython() |
|
175 | 181 | |
|
176 | 182 | # *** END HACK *** |
|
177 | 183 | ########################################################################### |
|
178 | 184 | |
|
179 | 185 | # Classes and functions |
|
180 | 186 | |
|
181 | 187 | def is_extension_module(filename): |
|
182 | 188 | """Return whether the given filename is an extension module. |
|
183 | 189 | |
|
184 | 190 | This simply checks that the extension is either .so or .pyd. |
|
185 | 191 | """ |
|
186 | 192 | return os.path.splitext(filename)[1].lower() in ('.so','.pyd') |
|
187 | 193 | |
|
188 | 194 | |
|
189 | 195 | class DocTestSkip(object): |
|
190 | 196 | """Object wrapper for doctests to be skipped.""" |
|
191 | 197 | |
|
192 | 198 | ds_skip = """Doctest to skip. |
|
193 | 199 | >>> 1 #doctest: +SKIP |
|
194 | 200 | """ |
|
195 | 201 | |
|
196 | 202 | def __init__(self,obj): |
|
197 | 203 | self.obj = obj |
|
198 | 204 | |
|
199 | 205 | def __getattribute__(self,key): |
|
200 | 206 | if key == '__doc__': |
|
201 | 207 | return DocTestSkip.ds_skip |
|
202 | 208 | else: |
|
203 | 209 | return getattr(object.__getattribute__(self,'obj'),key) |
|
204 | 210 | |
|
205 | 211 | # Modified version of the one in the stdlib, that fixes a python bug (doctests |
|
206 | 212 | # not found in extension modules, http://bugs.python.org/issue3158) |
|
207 | 213 | class DocTestFinder(doctest.DocTestFinder): |
|
208 | 214 | |
|
209 | 215 | def _from_module(self, module, object): |
|
210 | 216 | """ |
|
211 | 217 | Return true if the given object is defined in the given |
|
212 | 218 | module. |
|
213 | 219 | """ |
|
214 | 220 | if module is None: |
|
215 | 221 | return True |
|
216 | 222 | elif inspect.isfunction(object): |
|
217 | 223 | return module.__dict__ is object.func_globals |
|
218 | 224 | elif inspect.isbuiltin(object): |
|
219 | 225 | return module.__name__ == object.__module__ |
|
220 | 226 | elif inspect.isclass(object): |
|
221 | 227 | return module.__name__ == object.__module__ |
|
222 | 228 | elif inspect.ismethod(object): |
|
223 | 229 | # This one may be a bug in cython that fails to correctly set the |
|
224 | 230 | # __module__ attribute of methods, but since the same error is easy |
|
225 | 231 | # to make by extension code writers, having this safety in place |
|
226 | 232 | # isn't such a bad idea |
|
227 | 233 | return module.__name__ == object.im_class.__module__ |
|
228 | 234 | elif inspect.getmodule(object) is not None: |
|
229 | 235 | return module is inspect.getmodule(object) |
|
230 | 236 | elif hasattr(object, '__module__'): |
|
231 | 237 | return module.__name__ == object.__module__ |
|
232 | 238 | elif isinstance(object, property): |
|
233 | 239 | return True # [XX] no way not be sure. |
|
234 | 240 | else: |
|
235 | 241 | raise ValueError("object must be a class or function") |
|
236 | 242 | |
|
237 | 243 | def _find(self, tests, obj, name, module, source_lines, globs, seen): |
|
238 | 244 | """ |
|
239 | 245 | Find tests for the given object and any contained objects, and |
|
240 | 246 | add them to `tests`. |
|
241 | 247 | """ |
|
242 | 248 | |
|
243 | 249 | if hasattr(obj,"skip_doctest"): |
|
244 | 250 | #print 'SKIPPING DOCTEST FOR:',obj # dbg |
|
245 | 251 | obj = DocTestSkip(obj) |
|
246 | 252 | |
|
247 | 253 | doctest.DocTestFinder._find(self,tests, obj, name, module, |
|
248 | 254 | source_lines, globs, seen) |
|
249 | 255 | |
|
250 | 256 | # Below we re-run pieces of the above method with manual modifications, |
|
251 | 257 | # because the original code is buggy and fails to correctly identify |
|
252 | 258 | # doctests in extension modules. |
|
253 | 259 | |
|
254 | 260 | # Local shorthands |
|
255 | 261 | from inspect import isroutine, isclass, ismodule |
|
256 | 262 | |
|
257 | 263 | # Look for tests in a module's contained objects. |
|
258 | 264 | if inspect.ismodule(obj) and self._recurse: |
|
259 | 265 | for valname, val in obj.__dict__.items(): |
|
260 | 266 | valname1 = '%s.%s' % (name, valname) |
|
261 | 267 | if ( (isroutine(val) or isclass(val)) |
|
262 | 268 | and self._from_module(module, val) ): |
|
263 | 269 | |
|
264 | 270 | self._find(tests, val, valname1, module, source_lines, |
|
265 | 271 | globs, seen) |
|
266 | 272 | |
|
267 | 273 | # Look for tests in a class's contained objects. |
|
268 | 274 | if inspect.isclass(obj) and self._recurse: |
|
269 | 275 | #print 'RECURSE into class:',obj # dbg |
|
270 | 276 | for valname, val in obj.__dict__.items(): |
|
271 | 277 | # Special handling for staticmethod/classmethod. |
|
272 | 278 | if isinstance(val, staticmethod): |
|
273 | 279 | val = getattr(obj, valname) |
|
274 | 280 | if isinstance(val, classmethod): |
|
275 | 281 | val = getattr(obj, valname).im_func |
|
276 | 282 | |
|
277 | 283 | # Recurse to methods, properties, and nested classes. |
|
278 | 284 | if ((inspect.isfunction(val) or inspect.isclass(val) or |
|
279 | 285 | inspect.ismethod(val) or |
|
280 | 286 | isinstance(val, property)) and |
|
281 | 287 | self._from_module(module, val)): |
|
282 | 288 | valname = '%s.%s' % (name, valname) |
|
283 | 289 | self._find(tests, val, valname, module, source_lines, |
|
284 | 290 | globs, seen) |
|
285 | 291 | |
|
286 | 292 | |
|
287 | 293 | class IPDoctestOutputChecker(doctest.OutputChecker): |
|
288 | 294 | """Second-chance checker with support for random tests. |
|
289 | 295 | |
|
290 | 296 | If the default comparison doesn't pass, this checker looks in the expected |
|
291 | 297 | output string for flags that tell us to ignore the output. |
|
292 | 298 | """ |
|
293 | 299 | |
|
294 | 300 | random_re = re.compile(r'#\s*random\s+') |
|
295 | 301 | |
|
296 | 302 | def check_output(self, want, got, optionflags): |
|
297 | 303 | """Check output, accepting special markers embedded in the output. |
|
298 | 304 | |
|
299 | 305 | If the output didn't pass the default validation but the special string |
|
300 | 306 | '#random' is included, we accept it.""" |
|
301 | 307 | |
|
302 | 308 | # Let the original tester verify first, in case people have valid tests |
|
303 | 309 | # that happen to have a comment saying '#random' embedded in. |
|
304 | 310 | ret = doctest.OutputChecker.check_output(self, want, got, |
|
305 | 311 | optionflags) |
|
306 | 312 | if not ret and self.random_re.search(want): |
|
307 | 313 | #print >> sys.stderr, 'RANDOM OK:',want # dbg |
|
308 | 314 | return True |
|
309 | 315 | |
|
310 | 316 | return ret |
|
311 | 317 | |
|
312 | 318 | |
|
313 | 319 | class DocTestCase(doctests.DocTestCase): |
|
314 | 320 | """Proxy for DocTestCase: provides an address() method that |
|
315 | 321 | returns the correct address for the doctest case. Otherwise |
|
316 | 322 | acts as a proxy to the test case. To provide hints for address(), |
|
317 | 323 | an obj may also be passed -- this will be used as the test object |
|
318 | 324 | for purposes of determining the test address, if it is provided. |
|
319 | 325 | """ |
|
320 | 326 | |
|
321 | 327 | # Note: this method was taken from numpy's nosetester module. |
|
322 | 328 | |
|
323 | 329 | # Subclass nose.plugins.doctests.DocTestCase to work around a bug in |
|
324 | 330 | # its constructor that blocks non-default arguments from being passed |
|
325 | 331 | # down into doctest.DocTestCase |
|
326 | 332 | |
|
327 | 333 | def __init__(self, test, optionflags=0, setUp=None, tearDown=None, |
|
328 | 334 | checker=None, obj=None, result_var='_'): |
|
329 | 335 | self._result_var = result_var |
|
330 | 336 | doctests.DocTestCase.__init__(self, test, |
|
331 | 337 | optionflags=optionflags, |
|
332 | 338 | setUp=setUp, tearDown=tearDown, |
|
333 | 339 | checker=checker) |
|
334 | 340 | # Now we must actually copy the original constructor from the stdlib |
|
335 | 341 | # doctest class, because we can't call it directly and a bug in nose |
|
336 | 342 | # means it never gets passed the right arguments. |
|
337 | 343 | |
|
338 | 344 | self._dt_optionflags = optionflags |
|
339 | 345 | self._dt_checker = checker |
|
340 | 346 | self._dt_test = test |
|
341 | 347 | self._dt_setUp = setUp |
|
342 | 348 | self._dt_tearDown = tearDown |
|
343 | 349 | |
|
344 | 350 | # XXX - store this runner once in the object! |
|
345 | 351 | runner = IPDocTestRunner(optionflags=optionflags, |
|
346 | 352 | checker=checker, verbose=False) |
|
347 | 353 | self._dt_runner = runner |
|
348 | 354 | |
|
349 | 355 | |
|
350 | 356 | # Each doctest should remember what directory it was loaded from... |
|
351 | 357 | self._ori_dir = os.getcwd() |
|
352 | 358 | |
|
353 | 359 | # Modified runTest from the default stdlib |
|
354 | 360 | def runTest(self): |
|
355 | 361 | test = self._dt_test |
|
356 | 362 | runner = self._dt_runner |
|
357 | 363 | |
|
358 | 364 | old = sys.stdout |
|
359 | 365 | new = StringIO() |
|
360 | 366 | optionflags = self._dt_optionflags |
|
361 | 367 | |
|
362 | 368 | if not (optionflags & REPORTING_FLAGS): |
|
363 | 369 | # The option flags don't include any reporting flags, |
|
364 | 370 | # so add the default reporting flags |
|
365 | 371 | optionflags |= _unittest_reportflags |
|
366 | 372 | |
|
367 | 373 | try: |
|
368 | 374 | # Save our current directory and switch out to the one where the |
|
369 | 375 | # test was originally created, in case another doctest did a |
|
370 | 376 | # directory change. We'll restore this in the finally clause. |
|
371 | 377 | curdir = os.getcwd() |
|
372 | 378 | os.chdir(self._ori_dir) |
|
373 | 379 | |
|
374 | 380 | runner.DIVIDER = "-"*70 |
|
375 | 381 | failures, tries = runner.run(test,out=new.write, |
|
376 | 382 | clear_globs=False) |
|
377 | 383 | finally: |
|
378 | 384 | sys.stdout = old |
|
379 | 385 | os.chdir(curdir) |
|
380 | 386 | |
|
381 | 387 | if failures: |
|
382 | 388 | raise self.failureException(self.format_failure(new.getvalue())) |
|
383 | 389 | |
|
384 | 390 | def setUp(self): |
|
385 | 391 | """Modified test setup that syncs with ipython namespace""" |
|
386 | 392 | |
|
387 | 393 | if isinstance(self._dt_test.examples[0],IPExample): |
|
388 | 394 | # for IPython examples *only*, we swap the globals with the ipython |
|
389 | 395 | # namespace, after updating it with the globals (which doctest |
|
390 | 396 | # fills with the necessary info from the module being tested). |
|
391 | 397 | _ip.IP.user_ns.update(self._dt_test.globs) |
|
392 | 398 | self._dt_test.globs = _ip.IP.user_ns |
|
393 | 399 | |
|
394 | 400 | doctests.DocTestCase.setUp(self) |
|
395 | 401 | |
|
396 | 402 | |
|
397 | 403 | # A simple subclassing of the original with a different class name, so we can |
|
398 | 404 | # distinguish and treat differently IPython examples from pure python ones. |
|
399 | 405 | class IPExample(doctest.Example): pass |
|
400 | 406 | |
|
401 | 407 | |
|
402 | 408 | class IPExternalExample(doctest.Example): |
|
403 | 409 | """Doctest examples to be run in an external process.""" |
|
404 | 410 | |
|
405 | 411 | def __init__(self, source, want, exc_msg=None, lineno=0, indent=0, |
|
406 | 412 | options=None): |
|
407 | 413 | # Parent constructor |
|
408 | 414 | doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options) |
|
409 | 415 | |
|
410 | 416 | # An EXTRA newline is needed to prevent pexpect hangs |
|
411 | 417 | self.source += '\n' |
|
412 | 418 | |
|
413 | 419 | |
|
414 | 420 | class IPDocTestParser(doctest.DocTestParser): |
|
415 | 421 | """ |
|
416 | 422 | A class used to parse strings containing doctest examples. |
|
417 | 423 | |
|
418 | 424 | Note: This is a version modified to properly recognize IPython input and |
|
419 | 425 | convert any IPython examples into valid Python ones. |
|
420 | 426 | """ |
|
421 | 427 | # This regular expression is used to find doctest examples in a |
|
422 | 428 | # string. It defines three groups: `source` is the source code |
|
423 | 429 | # (including leading indentation and prompts); `indent` is the |
|
424 | 430 | # indentation of the first (PS1) line of the source code; and |
|
425 | 431 | # `want` is the expected output (including leading indentation). |
|
426 | 432 | |
|
427 | 433 | # Classic Python prompts or default IPython ones |
|
428 | 434 | _PS1_PY = r'>>>' |
|
429 | 435 | _PS2_PY = r'\.\.\.' |
|
430 | 436 | |
|
431 | 437 | _PS1_IP = r'In\ \[\d+\]:' |
|
432 | 438 | _PS2_IP = r'\ \ \ \.\.\.+:' |
|
433 | 439 | |
|
434 | 440 | _RE_TPL = r''' |
|
435 | 441 | # Source consists of a PS1 line followed by zero or more PS2 lines. |
|
436 | 442 | (?P<source> |
|
437 | 443 | (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line |
|
438 | 444 | (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines |
|
439 | 445 | \n? # a newline |
|
440 | 446 | # Want consists of any non-blank lines that do not start with PS1. |
|
441 | 447 | (?P<want> (?:(?![ ]*$) # Not a blank line |
|
442 | 448 | (?![ ]*%s) # Not a line starting with PS1 |
|
443 | 449 | (?![ ]*%s) # Not a line starting with PS2 |
|
444 | 450 | .*$\n? # But any other line |
|
445 | 451 | )*) |
|
446 | 452 | ''' |
|
447 | 453 | |
|
448 | 454 | _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY), |
|
449 | 455 | re.MULTILINE | re.VERBOSE) |
|
450 | 456 | |
|
451 | 457 | _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP), |
|
452 | 458 | re.MULTILINE | re.VERBOSE) |
|
453 | 459 | |
|
454 | 460 | # Mark a test as being fully random. In this case, we simply append the |
|
455 | 461 | # random marker ('#random') to each individual example's output. This way |
|
456 | 462 | # we don't need to modify any other code. |
|
457 | 463 | _RANDOM_TEST = re.compile(r'#\s*all-random\s+') |
|
458 | 464 | |
|
459 | 465 | # Mark tests to be executed in an external process - currently unsupported. |
|
460 | 466 | _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL') |
|
461 | 467 | |
|
462 | 468 | def ip2py(self,source): |
|
463 | 469 | """Convert input IPython source into valid Python.""" |
|
464 | 470 | out = [] |
|
465 | 471 | newline = out.append |
|
466 | 472 | #print 'IPSRC:\n',source,'\n###' # dbg |
|
467 | 473 | # The input source must be first stripped of all bracketing whitespace |
|
468 | 474 | # and turned into lines, so it looks to the parser like regular user |
|
469 | 475 | # input |
|
470 | 476 | for lnum,line in enumerate(source.strip().splitlines()): |
|
471 | 477 | newline(_ip.IP.prefilter(line,lnum>0)) |
|
472 | 478 | newline('') # ensure a closing newline, needed by doctest |
|
473 | 479 | #print "PYSRC:", '\n'.join(out) # dbg |
|
474 | 480 | return '\n'.join(out) |
|
475 | 481 | |
|
476 | 482 | def parse(self, string, name='<string>'): |
|
477 | 483 | """ |
|
478 | 484 | Divide the given string into examples and intervening text, |
|
479 | 485 | and return them as a list of alternating Examples and strings. |
|
480 | 486 | Line numbers for the Examples are 0-based. The optional |
|
481 | 487 | argument `name` is a name identifying this string, and is only |
|
482 | 488 | used for error messages. |
|
483 | 489 | """ |
|
484 | 490 | |
|
485 | 491 | #print 'Parse string:\n',string # dbg |
|
486 | 492 | |
|
487 | 493 | string = string.expandtabs() |
|
488 | 494 | # If all lines begin with the same indentation, then strip it. |
|
489 | 495 | min_indent = self._min_indent(string) |
|
490 | 496 | if min_indent > 0: |
|
491 | 497 | string = '\n'.join([l[min_indent:] for l in string.split('\n')]) |
|
492 | 498 | |
|
493 | 499 | output = [] |
|
494 | 500 | charno, lineno = 0, 0 |
|
495 | 501 | |
|
496 | 502 | # We make 'all random' tests by adding the '# random' mark to every |
|
497 | 503 | # block of output in the test. |
|
498 | 504 | if self._RANDOM_TEST.search(string): |
|
499 | 505 | random_marker = '\n# random' |
|
500 | 506 | else: |
|
501 | 507 | random_marker = '' |
|
502 | 508 | |
|
503 | 509 | # Whether to convert the input from ipython to python syntax |
|
504 | 510 | ip2py = False |
|
505 | 511 | # Find all doctest examples in the string. First, try them as Python |
|
506 | 512 | # examples, then as IPython ones |
|
507 | 513 | terms = list(self._EXAMPLE_RE_PY.finditer(string)) |
|
508 | 514 | if terms: |
|
509 | 515 | # Normal Python example |
|
510 | 516 | #print '-'*70 # dbg |
|
511 | 517 | #print 'PyExample, Source:\n',string # dbg |
|
512 | 518 | #print '-'*70 # dbg |
|
513 | 519 | Example = doctest.Example |
|
514 | 520 | else: |
|
515 | 521 | # It's an ipython example. Note that IPExamples are run |
|
516 | 522 | # in-process, so their syntax must be turned into valid python. |
|
517 | 523 | # IPExternalExamples are run out-of-process (via pexpect) so they |
|
518 | 524 | # don't need any filtering (a real ipython will be executing them). |
|
519 | 525 | terms = list(self._EXAMPLE_RE_IP.finditer(string)) |
|
520 | 526 | if self._EXTERNAL_IP.search(string): |
|
521 | 527 | #print '-'*70 # dbg |
|
522 | 528 | #print 'IPExternalExample, Source:\n',string # dbg |
|
523 | 529 | #print '-'*70 # dbg |
|
524 | 530 | Example = IPExternalExample |
|
525 | 531 | else: |
|
526 | 532 | #print '-'*70 # dbg |
|
527 | 533 | #print 'IPExample, Source:\n',string # dbg |
|
528 | 534 | #print '-'*70 # dbg |
|
529 | 535 | Example = IPExample |
|
530 | 536 | ip2py = True |
|
531 | 537 | |
|
532 | 538 | for m in terms: |
|
533 | 539 | # Add the pre-example text to `output`. |
|
534 | 540 | output.append(string[charno:m.start()]) |
|
535 | 541 | # Update lineno (lines before this example) |
|
536 | 542 | lineno += string.count('\n', charno, m.start()) |
|
537 | 543 | # Extract info from the regexp match. |
|
538 | 544 | (source, options, want, exc_msg) = \ |
|
539 | 545 | self._parse_example(m, name, lineno,ip2py) |
|
540 | 546 | |
|
541 | 547 | # Append the random-output marker (it defaults to empty in most |
|
542 | 548 | # cases, it's only non-empty for 'all-random' tests): |
|
543 | 549 | want += random_marker |
|
544 | 550 | |
|
545 | 551 | if Example is IPExternalExample: |
|
546 | 552 | options[doctest.NORMALIZE_WHITESPACE] = True |
|
547 | 553 | want += '\n' |
|
548 | 554 | |
|
549 | 555 | # Create an Example, and add it to the list. |
|
550 | 556 | if not self._IS_BLANK_OR_COMMENT(source): |
|
551 | 557 | output.append(Example(source, want, exc_msg, |
|
552 | 558 | lineno=lineno, |
|
553 | 559 | indent=min_indent+len(m.group('indent')), |
|
554 | 560 | options=options)) |
|
555 | 561 | # Update lineno (lines inside this example) |
|
556 | 562 | lineno += string.count('\n', m.start(), m.end()) |
|
557 | 563 | # Update charno. |
|
558 | 564 | charno = m.end() |
|
559 | 565 | # Add any remaining post-example text to `output`. |
|
560 | 566 | output.append(string[charno:]) |
|
561 | 567 | return output |
|
562 | 568 | |
|
563 | 569 | def _parse_example(self, m, name, lineno,ip2py=False): |
|
564 | 570 | """ |
|
565 | 571 | Given a regular expression match from `_EXAMPLE_RE` (`m`), |
|
566 | 572 | return a pair `(source, want)`, where `source` is the matched |
|
567 | 573 | example's source code (with prompts and indentation stripped); |
|
568 | 574 | and `want` is the example's expected output (with indentation |
|
569 | 575 | stripped). |
|
570 | 576 | |
|
571 | 577 | `name` is the string's name, and `lineno` is the line number |
|
572 | 578 | where the example starts; both are used for error messages. |
|
573 | 579 | |
|
574 | 580 | Optional: |
|
575 | 581 | `ip2py`: if true, filter the input via IPython to convert the syntax |
|
576 | 582 | into valid python. |
|
577 | 583 | """ |
|
578 | 584 | |
|
579 | 585 | # Get the example's indentation level. |
|
580 | 586 | indent = len(m.group('indent')) |
|
581 | 587 | |
|
582 | 588 | # Divide source into lines; check that they're properly |
|
583 | 589 | # indented; and then strip their indentation & prompts. |
|
584 | 590 | source_lines = m.group('source').split('\n') |
|
585 | 591 | |
|
586 | 592 | # We're using variable-length input prompts |
|
587 | 593 | ps1 = m.group('ps1') |
|
588 | 594 | ps2 = m.group('ps2') |
|
589 | 595 | ps1_len = len(ps1) |
|
590 | 596 | |
|
591 | 597 | self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len) |
|
592 | 598 | if ps2: |
|
593 | 599 | self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno) |
|
594 | 600 | |
|
595 | 601 | source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines]) |
|
596 | 602 | |
|
597 | 603 | if ip2py: |
|
598 | 604 | # Convert source input from IPython into valid Python syntax |
|
599 | 605 | source = self.ip2py(source) |
|
600 | 606 | |
|
601 | 607 | # Divide want into lines; check that it's properly indented; and |
|
602 | 608 | # then strip the indentation. Spaces before the last newline should |
|
603 | 609 | # be preserved, so plain rstrip() isn't good enough. |
|
604 | 610 | want = m.group('want') |
|
605 | 611 | want_lines = want.split('\n') |
|
606 | 612 | if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): |
|
607 | 613 | del want_lines[-1] # forget final newline & spaces after it |
|
608 | 614 | self._check_prefix(want_lines, ' '*indent, name, |
|
609 | 615 | lineno + len(source_lines)) |
|
610 | 616 | |
|
611 | 617 | # Remove ipython output prompt that might be present in the first line |
|
612 | 618 | want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0]) |
|
613 | 619 | |
|
614 | 620 | want = '\n'.join([wl[indent:] for wl in want_lines]) |
|
615 | 621 | |
|
616 | 622 | # If `want` contains a traceback message, then extract it. |
|
617 | 623 | m = self._EXCEPTION_RE.match(want) |
|
618 | 624 | if m: |
|
619 | 625 | exc_msg = m.group('msg') |
|
620 | 626 | else: |
|
621 | 627 | exc_msg = None |
|
622 | 628 | |
|
623 | 629 | # Extract options from the source. |
|
624 | 630 | options = self._find_options(source, name, lineno) |
|
625 | 631 | |
|
626 | 632 | return source, options, want, exc_msg |
|
627 | 633 | |
|
628 | 634 | def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len): |
|
629 | 635 | """ |
|
630 | 636 | Given the lines of a source string (including prompts and |
|
631 | 637 | leading indentation), check to make sure that every prompt is |
|
632 | 638 | followed by a space character. If any line is not followed by |
|
633 | 639 | a space character, then raise ValueError. |
|
634 | 640 | |
|
635 | 641 | Note: IPython-modified version which takes the input prompt length as a |
|
636 | 642 | parameter, so that prompts of variable length can be dealt with. |
|
637 | 643 | """ |
|
638 | 644 | space_idx = indent+ps1_len |
|
639 | 645 | min_len = space_idx+1 |
|
640 | 646 | for i, line in enumerate(lines): |
|
641 | 647 | if len(line) >= min_len and line[space_idx] != ' ': |
|
642 | 648 | raise ValueError('line %r of the docstring for %s ' |
|
643 | 649 | 'lacks blank after %s: %r' % |
|
644 | 650 | (lineno+i+1, name, |
|
645 | 651 | line[indent:space_idx], line)) |
|
646 | 652 | |
|
647 | 653 | |
|
648 | 654 | SKIP = doctest.register_optionflag('SKIP') |
|
649 | 655 | |
|
650 | 656 | |
|
651 | 657 | class IPDocTestRunner(doctest.DocTestRunner,object): |
|
652 | 658 | """Test runner that synchronizes the IPython namespace with test globals. |
|
653 | 659 | """ |
|
654 | 660 | |
|
655 | 661 | def run(self, test, compileflags=None, out=None, clear_globs=True): |
|
656 | 662 | |
|
657 | 663 | # Hack: ipython needs access to the execution context of the example, |
|
658 | 664 | # so that it can propagate user variables loaded by %run into |
|
659 | 665 | # test.globs. We put them here into our modified %run as a function |
|
660 | 666 | # attribute. Our new %run will then only make the namespace update |
|
661 | 667 | # when called (rather than unconconditionally updating test.globs here |
|
662 | 668 | # for all examples, most of which won't be calling %run anyway). |
|
663 | 669 | _run_ns_sync.test_globs = test.globs |
|
664 | 670 | _run_ns_sync.test_filename = test.filename |
|
665 | 671 | |
|
666 | 672 | return super(IPDocTestRunner,self).run(test, |
|
667 | 673 | compileflags,out,clear_globs) |
|
668 | 674 | |
|
669 | 675 | |
|
670 | 676 | class DocFileCase(doctest.DocFileCase): |
|
671 | 677 | """Overrides to provide filename |
|
672 | 678 | """ |
|
673 | 679 | def address(self): |
|
674 | 680 | return (self._dt_test.filename, None, None) |
|
675 | 681 | |
|
676 | 682 | |
|
677 | 683 | class ExtensionDoctest(doctests.Doctest): |
|
678 | 684 | """Nose Plugin that supports doctests in extension modules. |
|
679 | 685 | """ |
|
680 | 686 | name = 'extdoctest' # call nosetests with --with-extdoctest |
|
681 | 687 | enabled = True |
|
682 | 688 | |
|
683 | 689 | def __init__(self,exclude_patterns=None): |
|
684 | 690 | """Create a new ExtensionDoctest plugin. |
|
685 | 691 | |
|
686 | 692 | Parameters |
|
687 | 693 | ---------- |
|
688 | 694 | |
|
689 | 695 | exclude_patterns : sequence of strings, optional |
|
690 | 696 | These patterns are compiled as regular expressions, subsequently used |
|
691 | 697 | to exclude any filename which matches them from inclusion in the test |
|
692 | 698 | suite (using pattern.search(), NOT pattern.match() ). |
|
693 | 699 | """ |
|
700 | ||
|
694 | 701 | if exclude_patterns is None: |
|
695 | 702 | exclude_patterns = [] |
|
696 | 703 | self.exclude_patterns = map(re.compile,exclude_patterns) |
|
697 | 704 | doctests.Doctest.__init__(self) |
|
698 | 705 | |
|
699 | 706 | def options(self, parser, env=os.environ): |
|
700 | 707 | Plugin.options(self, parser, env) |
|
701 | 708 | parser.add_option('--doctest-tests', action='store_true', |
|
702 | 709 | dest='doctest_tests', |
|
703 | 710 | default=env.get('NOSE_DOCTEST_TESTS',True), |
|
704 | 711 | help="Also look for doctests in test modules. " |
|
705 | 712 | "Note that classes, methods and functions should " |
|
706 | 713 | "have either doctests or non-doctest tests, " |
|
707 | 714 | "not both. [NOSE_DOCTEST_TESTS]") |
|
708 | 715 | parser.add_option('--doctest-extension', action="append", |
|
709 | 716 | dest="doctestExtension", |
|
710 | 717 | help="Also look for doctests in files with " |
|
711 | 718 | "this extension [NOSE_DOCTEST_EXTENSION]") |
|
712 | 719 | # Set the default as a list, if given in env; otherwise |
|
713 | 720 | # an additional value set on the command line will cause |
|
714 | 721 | # an error. |
|
715 | 722 | env_setting = env.get('NOSE_DOCTEST_EXTENSION') |
|
716 | 723 | if env_setting is not None: |
|
717 | 724 | parser.set_defaults(doctestExtension=tolist(env_setting)) |
|
718 | 725 | |
|
719 | 726 | |
|
720 | 727 | def configure(self, options, config): |
|
721 | 728 | Plugin.configure(self, options, config) |
|
722 | 729 | self.doctest_tests = options.doctest_tests |
|
723 | 730 | self.extension = tolist(options.doctestExtension) |
|
724 | 731 | |
|
725 | 732 | self.parser = doctest.DocTestParser() |
|
726 | 733 | self.finder = DocTestFinder() |
|
727 | 734 | self.checker = IPDoctestOutputChecker() |
|
728 | 735 | self.globs = None |
|
729 | 736 | self.extraglobs = None |
|
730 | 737 | |
|
731 | 738 | |
|
732 | 739 | def loadTestsFromExtensionModule(self,filename): |
|
733 | 740 | bpath,mod = os.path.split(filename) |
|
734 | 741 | modname = os.path.splitext(mod)[0] |
|
735 | 742 | try: |
|
736 | 743 | sys.path.append(bpath) |
|
737 | 744 | module = __import__(modname) |
|
738 | 745 | tests = list(self.loadTestsFromModule(module)) |
|
739 | 746 | finally: |
|
740 | 747 | sys.path.pop() |
|
741 | 748 | return tests |
|
742 | 749 | |
|
743 | 750 | # NOTE: the method below is almost a copy of the original one in nose, with |
|
744 | 751 | # a few modifications to control output checking. |
|
745 | 752 | |
|
746 | 753 | def loadTestsFromModule(self, module): |
|
747 | 754 | #print '*** ipdoctest - lTM',module # dbg |
|
748 | 755 | |
|
749 | 756 | if not self.matches(module.__name__): |
|
750 | 757 | log.debug("Doctest doesn't want module %s", module) |
|
751 | 758 | return |
|
752 | 759 | |
|
753 | 760 | tests = self.finder.find(module,globs=self.globs, |
|
754 | 761 | extraglobs=self.extraglobs) |
|
755 | 762 | if not tests: |
|
756 | 763 | return |
|
757 | 764 | |
|
758 | 765 | # always use whitespace and ellipsis options |
|
759 | 766 | optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS |
|
760 | 767 | |
|
761 | 768 | tests.sort() |
|
762 | 769 | module_file = module.__file__ |
|
763 | 770 | if module_file[-4:] in ('.pyc', '.pyo'): |
|
764 | 771 | module_file = module_file[:-1] |
|
765 | 772 | for test in tests: |
|
766 | 773 | if not test.examples: |
|
767 | 774 | continue |
|
768 | 775 | if not test.filename: |
|
769 | 776 | test.filename = module_file |
|
770 | 777 | |
|
771 | 778 | yield DocTestCase(test, |
|
772 | 779 | optionflags=optionflags, |
|
773 | 780 | checker=self.checker) |
|
774 | 781 | |
|
775 | 782 | |
|
776 | 783 | def loadTestsFromFile(self, filename): |
|
777 | 784 | if is_extension_module(filename): |
|
778 | 785 | for t in self.loadTestsFromExtensionModule(filename): |
|
779 | 786 | yield t |
|
780 | 787 | else: |
|
781 | 788 | if self.extension and anyp(filename.endswith, self.extension): |
|
782 | 789 | name = os.path.basename(filename) |
|
783 | 790 | dh = open(filename) |
|
784 | 791 | try: |
|
785 | 792 | doc = dh.read() |
|
786 | 793 | finally: |
|
787 | 794 | dh.close() |
|
788 | 795 | test = self.parser.get_doctest( |
|
789 | 796 | doc, globs={'__file__': filename}, name=name, |
|
790 | 797 | filename=filename, lineno=0) |
|
791 | 798 | if test.examples: |
|
792 | 799 | #print 'FileCase:',test.examples # dbg |
|
793 | 800 | yield DocFileCase(test) |
|
794 | 801 | else: |
|
795 | 802 | yield False # no tests to load |
|
796 | 803 | |
|
797 | 804 | def wantFile(self,filename): |
|
798 | 805 | """Return whether the given filename should be scanned for tests. |
|
799 | 806 | |
|
800 | 807 | Modified version that accepts extension modules as valid containers for |
|
801 | 808 | doctests. |
|
802 | 809 | """ |
|
803 | 810 | #print '*** ipdoctest- wantFile:',filename # dbg |
|
804 | 811 | |
|
805 | 812 | for pat in self.exclude_patterns: |
|
806 | 813 | if pat.search(filename): |
|
807 | 814 | #print '###>>> SKIP:',filename # dbg |
|
808 | 815 | return False |
|
809 | 816 | |
|
810 | 817 | if is_extension_module(filename): |
|
811 | 818 | return True |
|
812 | 819 | else: |
|
813 | 820 | return doctests.Doctest.wantFile(self,filename) |
|
814 | 821 | |
|
815 | 822 | |
|
816 | 823 | class IPythonDoctest(ExtensionDoctest): |
|
817 | 824 | """Nose Plugin that supports doctests in extension modules. |
|
818 | 825 | """ |
|
819 | 826 | name = 'ipdoctest' # call nosetests with --with-ipdoctest |
|
820 | 827 | enabled = True |
|
821 | 828 | |
|
822 | 829 | def makeTest(self, obj, parent): |
|
823 | 830 | """Look for doctests in the given object, which will be a |
|
824 | 831 | function, method or class. |
|
825 | 832 | """ |
|
826 | 833 | # always use whitespace and ellipsis options |
|
827 | 834 | optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS |
|
828 | 835 | |
|
829 | 836 | doctests = self.finder.find(obj, module=getmodule(parent)) |
|
830 | 837 | if doctests: |
|
831 | 838 | for test in doctests: |
|
832 | 839 | if len(test.examples) == 0: |
|
833 | 840 | continue |
|
834 | 841 | |
|
835 | 842 | yield DocTestCase(test, obj=obj, |
|
836 | 843 | optionflags=optionflags, |
|
837 | 844 | checker=self.checker) |
|
838 | 845 | |
|
839 | def configure(self, options, config): | |
|
846 | def options(self, parser, env=os.environ): | |
|
847 | Plugin.options(self, parser, env) | |
|
848 | parser.add_option('--ipdoctest-tests', action='store_true', | |
|
849 | dest='ipdoctest_tests', | |
|
850 | default=env.get('NOSE_IPDOCTEST_TESTS',True), | |
|
851 | help="Also look for doctests in test modules. " | |
|
852 | "Note that classes, methods and functions should " | |
|
853 | "have either doctests or non-doctest tests, " | |
|
854 | "not both. [NOSE_IPDOCTEST_TESTS]") | |
|
855 | parser.add_option('--ipdoctest-extension', action="append", | |
|
856 | dest="ipdoctest_extension", | |
|
857 | help="Also look for doctests in files with " | |
|
858 | "this extension [NOSE_IPDOCTEST_EXTENSION]") | |
|
859 | # Set the default as a list, if given in env; otherwise | |
|
860 | # an additional value set on the command line will cause | |
|
861 | # an error. | |
|
862 | env_setting = env.get('NOSE_IPDOCTEST_EXTENSION') | |
|
863 | if env_setting is not None: | |
|
864 | parser.set_defaults(ipdoctest_extension=tolist(env_setting)) | |
|
840 | 865 | |
|
866 | def configure(self, options, config): | |
|
841 | 867 | Plugin.configure(self, options, config) |
|
842 | self.doctest_tests = options.doctest_tests | |
|
843 |
self.extension = tolist(options.doctest |
|
|
868 | self.doctest_tests = options.ipdoctest_tests | |
|
869 | self.extension = tolist(options.ipdoctest_extension) | |
|
844 | 870 | |
|
845 | 871 | self.parser = IPDocTestParser() |
|
846 | 872 | self.finder = DocTestFinder(parser=self.parser) |
|
847 | 873 | self.checker = IPDoctestOutputChecker() |
|
848 | 874 | self.globs = None |
|
849 | 875 | self.extraglobs = None |
|
850 |
@@ -1,17 +1,18 b'' | |||
|
1 | 1 | """Simple script to show reference holding behavior. |
|
2 | 2 | |
|
3 | 3 | This is used by a companion test case. |
|
4 | 4 | """ |
|
5 | 5 | |
|
6 | 6 | import gc |
|
7 | 7 | |
|
8 | 8 | class C(object): |
|
9 | def __del__(self): | |
|
10 | print 'deleting object...' | |
|
9 | def __del__(self): | |
|
10 | pass | |
|
11 | #print 'deleting object...' # dbg | |
|
11 | 12 | |
|
12 | 13 | c = C() |
|
13 | 14 | |
|
14 | 15 | c_refs = gc.get_referrers(c) |
|
15 | 16 | ref_ids = map(id,c_refs) |
|
16 | 17 | |
|
17 | 18 | print 'c referrers:',map(type,c_refs) |
@@ -1,51 +1,48 b'' | |||
|
1 | 1 | """Some simple tests for the plugin while running scripts. |
|
2 | 2 | """ |
|
3 | 3 | # Module imports |
|
4 | 4 | # Std lib |
|
5 | 5 | import inspect |
|
6 | 6 | |
|
7 | 7 | # Our own |
|
8 | 8 | from IPython.testing import decorators as dec |
|
9 | 9 | |
|
10 | 10 | #----------------------------------------------------------------------------- |
|
11 | 11 | # Testing functions |
|
12 | 12 | |
|
13 | 13 | def test_trivial(): |
|
14 | 14 | """A trivial passing test.""" |
|
15 | 15 | pass |
|
16 | 16 | |
|
17 | 17 | def doctest_run(): |
|
18 | 18 | """Test running a trivial script. |
|
19 | 19 | |
|
20 | 20 | In [13]: run simplevars.py |
|
21 | 21 | x is: 1 |
|
22 | 22 | """ |
|
23 | 23 | |
|
24 | 24 | def doctest_runvars(): |
|
25 | 25 | """Test that variables defined in scripts get loaded correcly via %run. |
|
26 | 26 | |
|
27 | 27 | In [13]: run simplevars.py |
|
28 | 28 | x is: 1 |
|
29 | 29 | |
|
30 | 30 | In [14]: x |
|
31 | 31 | Out[14]: 1 |
|
32 | 32 | """ |
|
33 | 33 | |
|
34 | 34 | def doctest_ivars(): |
|
35 | 35 | """Test that variables defined interactively are picked up. |
|
36 | 36 | In [5]: zz=1 |
|
37 | 37 | |
|
38 | 38 | In [6]: zz |
|
39 | 39 | Out[6]: 1 |
|
40 | 40 | """ |
|
41 | 41 | |
|
42 | @dec.skip_doctest | |
|
42 | #@dec.skip_doctest | |
|
43 | 43 | def doctest_refs(): |
|
44 | 44 | """DocTest reference holding issues when running scripts. |
|
45 | 45 | |
|
46 | 46 | In [32]: run show_refs.py |
|
47 | 47 | c referrers: [<type 'dict'>] |
|
48 | ||
|
49 | In [33]: map(type,gc.get_referrers(c)) | |
|
50 | Out[33]: [<type 'dict'>] | |
|
51 | 48 | """ |
@@ -1,34 +1,34 b'' | |||
|
1 | 1 | """Test code for https://bugs.launchpad.net/ipython/+bug/239054 |
|
2 | 2 | |
|
3 | 3 | WARNING: this script exits IPython! It MUST be run in a subprocess. |
|
4 | 4 | |
|
5 | 5 | When you run the following script from CPython it prints: |
|
6 | 6 | __init__ is here |
|
7 | 7 | __del__ is here |
|
8 | 8 | |
|
9 | 9 | and creates the __del__.txt file |
|
10 | 10 | |
|
11 | 11 | When you run it from IPython it prints: |
|
12 | 12 | __init__ is here |
|
13 | 13 | |
|
14 | 14 | When you exit() or Exit from IPython neothing is printed and no file is created |
|
15 | 15 | (the file thing is to make sure __del__ is really never called and not that |
|
16 | 16 | just the output is eaten). |
|
17 | 17 | |
|
18 | 18 | Note that if you call %reset in IPython then everything is Ok. |
|
19 | 19 | |
|
20 | 20 | IPython should do the equivalent of %reset and release all the references it |
|
21 | 21 | holds before exit. This behavior is important when working with binding objects |
|
22 | 22 | that rely on __del__. If the current behavior has some use case then I suggest |
|
23 | 23 | to add a configuration option to IPython to control it. |
|
24 | 24 | """ |
|
25 | 25 | import sys |
|
26 | 26 | |
|
27 | 27 | class A(object): |
|
28 | 28 | def __del__(self): |
|
29 | print 'object A deleted' | |
|
29 | print 'obj_del.py: object A deleted' | |
|
30 | 30 | |
|
31 | 31 | a = A() |
|
32 | 32 | |
|
33 | 33 | # Now, we force an exit, the caller will check that the del printout was given |
|
34 | 34 | _ip.IP.ask_exit() |
@@ -1,26 +1,27 b'' | |||
|
1 | 1 | """Simple script to instantiate a class for testing %run""" |
|
2 | 2 | |
|
3 | 3 | import sys |
|
4 | 4 | |
|
5 | 5 | # An external test will check that calls to f() work after %run |
|
6 | 6 | class foo: pass |
|
7 | 7 | |
|
8 | 8 | def f(): |
|
9 | 9 | return foo() |
|
10 | 10 | |
|
11 | 11 | # We also want to ensure that while objects remain available for immediate |
|
12 | 12 | # access, objects from *previous* runs of the same script get collected, to |
|
13 | 13 | # avoid accumulating massive amounts of old references. |
|
14 | 14 | class C(object): |
|
15 | 15 | def __init__(self,name): |
|
16 | 16 | self.name = name |
|
17 | 17 | |
|
18 | 18 | def __del__(self): |
|
19 |
print ' |
|
|
19 | print 'tclass.py: deleting object:',self.name | |
|
20 | 20 | |
|
21 | 21 | try: |
|
22 | 22 | name = sys.argv[1] |
|
23 | 23 | except IndexError: |
|
24 | 24 | pass |
|
25 | 25 | else: |
|
26 | c = C(name) | |
|
26 | if name.startswith('C'): | |
|
27 | c = C(name) |
@@ -1,17 +1,68 b'' | |||
|
1 | 1 | """Tests for the key iplib module, where the main ipython class is defined. |
|
2 | 2 | """ |
|
3 | #----------------------------------------------------------------------------- | |
|
4 | # Module imports | |
|
5 | #----------------------------------------------------------------------------- | |
|
3 | 6 | |
|
7 | # stdlib | |
|
8 | import os | |
|
9 | import shutil | |
|
10 | import tempfile | |
|
11 | ||
|
12 | # third party | |
|
4 | 13 | import nose.tools as nt |
|
5 | 14 | |
|
15 | # our own packages | |
|
16 | from IPython import iplib | |
|
17 | ||
|
18 | #----------------------------------------------------------------------------- | |
|
19 | # Globals | |
|
20 | #----------------------------------------------------------------------------- | |
|
21 | ||
|
22 | # Useful global ipapi object and main IPython one. Unfortunately we have a | |
|
23 | # long precedent of carrying the 'ipapi' global object which is injected into | |
|
24 | # the system namespace as _ip, but that keeps a pointer to the actual IPython | |
|
25 | # InteractiveShell instance, which is named IP. Since in testing we do need | |
|
26 | # access to the real thing (we want to probe beyond what ipapi exposes), make | |
|
27 | # here a global reference to each. In general, things that are exposed by the | |
|
28 | # ipapi instance should be read from there, but we also will often need to use | |
|
29 | # the actual IPython one. | |
|
30 | ||
|
31 | ip = _ip # This is the ipapi instance | |
|
32 | IP = ip.IP # This is the actual IPython shell 'raw' object. | |
|
33 | ||
|
34 | #----------------------------------------------------------------------------- | |
|
35 | # Test functions | |
|
36 | #----------------------------------------------------------------------------- | |
|
6 | 37 | |
|
7 | 38 | def test_reset(): |
|
8 | 39 | """reset must clear most namespaces.""" |
|
9 | ip = _ip.IP | |
|
10 | ip.reset() # first, it should run without error | |
|
40 | IP.reset() # first, it should run without error | |
|
11 | 41 | # Then, check that most namespaces end up empty |
|
12 |
for ns in |
|
|
13 |
if ns is |
|
|
42 | for ns in IP.ns_refs_table: | |
|
43 | if ns is IP.user_ns: | |
|
14 | 44 | # The user namespace is reset with some data, so we can't check for |
|
15 | 45 | # it being empty |
|
16 | 46 | continue |
|
17 | 47 | nt.assert_equals(len(ns),0) |
|
48 | ||
|
49 | ||
|
50 | # make sure that user_setup can be run re-entrantly in 'install' mode. | |
|
51 | def test_user_setup(): | |
|
52 | # use a lambda to pass kwargs to the generator | |
|
53 | user_setup = lambda a,k: iplib.user_setup(*a,**k) | |
|
54 | kw = dict(mode='install', interactive=False) | |
|
55 | ||
|
56 | # Call the user setup and verify that the directory exists | |
|
57 | yield user_setup, (ip.options.ipythondir,''), kw | |
|
58 | yield os.path.isdir, ip.options.ipythondir | |
|
59 | ||
|
60 | # Now repeat the operation with a non-existent directory. Check both that | |
|
61 | # the call succeeds and that the directory is created. | |
|
62 | tmpdir = tempfile.mktemp(prefix='ipython-test-') | |
|
63 | try: | |
|
64 | yield user_setup, (tmpdir,''), kw | |
|
65 | yield os.path.isdir, tmpdir | |
|
66 | finally: | |
|
67 | # In this case, clean up the temp dir once done | |
|
68 | shutil.rmtree(tmpdir) |
@@ -1,135 +1,151 b'' | |||
|
1 | 1 | """Tests for various magic functions. |
|
2 | 2 | |
|
3 | 3 | Needs to be run by nose (to make ipython session available). |
|
4 | 4 | """ |
|
5 | 5 | |
|
6 | 6 | # Standard library imports |
|
7 | 7 | import os |
|
8 | 8 | import sys |
|
9 | 9 | |
|
10 | 10 | # Third-party imports |
|
11 | 11 | import nose.tools as nt |
|
12 | 12 | |
|
13 | 13 | # From our own code |
|
14 | 14 | from IPython.testing import decorators as dec |
|
15 | 15 | |
|
16 | 16 | #----------------------------------------------------------------------------- |
|
17 | 17 | # Test functions begin |
|
18 | 18 | |
|
19 | 19 | def test_rehashx(): |
|
20 | 20 | # clear up everything |
|
21 | 21 | _ip.IP.alias_table.clear() |
|
22 | 22 | del _ip.db['syscmdlist'] |
|
23 | 23 | |
|
24 | 24 | _ip.magic('rehashx') |
|
25 | 25 | # Practically ALL ipython development systems will have more than 10 aliases |
|
26 | 26 | |
|
27 | 27 | assert len(_ip.IP.alias_table) > 10 |
|
28 | 28 | for key, val in _ip.IP.alias_table.items(): |
|
29 | 29 | # we must strip dots from alias names |
|
30 | 30 | assert '.' not in key |
|
31 | 31 | |
|
32 | 32 | # rehashx must fill up syscmdlist |
|
33 | 33 | scoms = _ip.db['syscmdlist'] |
|
34 | 34 | assert len(scoms) > 10 |
|
35 | 35 | |
|
36 | 36 | |
|
37 | 37 | def doctest_run_ns(): |
|
38 | 38 | """Classes declared %run scripts must be instantiable afterwards. |
|
39 | 39 | |
|
40 | In [11]: run tclass | |
|
40 | In [11]: run tclass foo | |
|
41 | 41 | |
|
42 | 42 | In [12]: isinstance(f(),foo) |
|
43 | 43 | Out[12]: True |
|
44 | 44 | """ |
|
45 | 45 | |
|
46 | 46 | |
|
47 | 47 | def doctest_run_ns2(): |
|
48 | 48 | """Classes declared %run scripts must be instantiable afterwards. |
|
49 | 49 | |
|
50 |
In [ |
|
|
50 | In [4]: run tclass C-first_pass | |
|
51 | 51 | |
|
52 |
In [ |
|
|
53 | ||
|
54 | In [5]: run tclass second_pass | |
|
55 | Deleting object: first_pass | |
|
52 | In [5]: run tclass C-second_pass | |
|
53 | tclass.py: deleting object: C-first_pass | |
|
56 | 54 | """ |
|
57 | 55 | |
|
58 | 56 | |
|
59 | 57 | def doctest_hist_f(): |
|
60 | 58 | """Test %hist -f with temporary filename. |
|
61 | 59 | |
|
62 | 60 | In [9]: import tempfile |
|
63 | 61 | |
|
64 | 62 | In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-') |
|
65 | 63 | |
|
66 | 64 | In [11]: %history -n -f $tfile 3 |
|
67 | 65 | """ |
|
68 | 66 | |
|
69 | 67 | |
|
70 | 68 | def doctest_hist_r(): |
|
71 | 69 | """Test %hist -r |
|
72 | 70 | |
|
73 | 71 | XXX - This test is not recording the output correctly. Not sure why... |
|
74 | 72 | |
|
75 | 73 | In [6]: x=1 |
|
76 | 74 | |
|
77 | 75 | In [7]: hist -n -r 2 |
|
78 | 76 | x=1 # random |
|
79 | 77 | hist -n -r 2 # random |
|
80 | 78 | """ |
|
81 | 79 | |
|
82 | 80 | |
|
83 | 81 | def test_obj_del(): |
|
84 | 82 | """Test that object's __del__ methods are called on exit.""" |
|
85 | 83 | test_dir = os.path.dirname(__file__) |
|
86 | 84 | del_file = os.path.join(test_dir,'obj_del.py') |
|
87 | 85 | out = _ip.IP.getoutput('ipython %s' % del_file) |
|
88 | nt.assert_equals(out,'object A deleted') | |
|
86 | nt.assert_equals(out,'obj_del.py: object A deleted') | |
|
89 | 87 | |
|
90 | 88 | |
|
91 | 89 | def test_shist(): |
|
92 | 90 | # Simple tests of ShadowHist class - test generator. |
|
93 | 91 | import os, shutil, tempfile |
|
94 | 92 | |
|
95 | 93 | from IPython.Extensions import pickleshare |
|
96 | 94 | from IPython.history import ShadowHist |
|
97 | 95 | |
|
98 | 96 | tfile = tempfile.mktemp('','tmp-ipython-') |
|
99 | 97 | |
|
100 | 98 | db = pickleshare.PickleShareDB(tfile) |
|
101 | 99 | s = ShadowHist(db) |
|
102 | 100 | s.add('hello') |
|
103 | 101 | s.add('world') |
|
104 | 102 | s.add('hello') |
|
105 | 103 | s.add('hello') |
|
106 | 104 | s.add('karhu') |
|
107 | 105 | |
|
108 | 106 | yield nt.assert_equals,s.all(),[(1, 'hello'), (2, 'world'), (3, 'karhu')] |
|
109 | 107 | |
|
110 | 108 | yield nt.assert_equal,s.get(2),'world' |
|
111 | 109 | |
|
112 | 110 | shutil.rmtree(tfile) |
|
113 | 111 | |
|
114 | 112 | @dec.skipif_not_numpy |
|
115 | 113 | def test_numpy_clear_array_undec(): |
|
116 | 114 | _ip.ex('import numpy as np') |
|
117 | 115 | _ip.ex('a = np.empty(2)') |
|
118 | 116 | |
|
119 | 117 | yield nt.assert_true,'a' in _ip.user_ns |
|
120 | 118 | _ip.magic('clear array') |
|
121 | 119 | yield nt.assert_false,'a' in _ip.user_ns |
|
122 | 120 | |
|
123 | 121 | |
|
124 | 122 | @dec.skip() |
|
125 | 123 | def test_fail_dec(*a,**k): |
|
126 | 124 | yield nt.assert_true, False |
|
127 | 125 | |
|
128 | 126 | @dec.skip('This one shouldn not run') |
|
129 | 127 | def test_fail_dec2(*a,**k): |
|
130 | 128 | yield nt.assert_true, False |
|
131 | 129 | |
|
132 | 130 | @dec.skipknownfailure |
|
133 | 131 | def test_fail_dec3(*a,**k): |
|
134 | 132 | yield nt.assert_true, False |
|
135 | 133 | |
|
134 | ||
|
135 | def doctest_refbug(): | |
|
136 | """Very nasty problem with references held by multiple runs of a script. | |
|
137 | See: https://bugs.launchpad.net/ipython/+bug/269966 | |
|
138 | ||
|
139 | In [1]: _ip.IP.clear_main_mod_cache() | |
|
140 | ||
|
141 | In [2]: run refbug | |
|
142 | ||
|
143 | In [3]: call_f() | |
|
144 | lowercased: hello | |
|
145 | ||
|
146 | In [4]: run refbug | |
|
147 | ||
|
148 | In [5]: call_f() | |
|
149 | lowercased: hello | |
|
150 | lowercased: hello | |
|
151 | """ |
@@ -1,398 +1,401 b'' | |||
|
1 | 1 | .. _changes: |
|
2 | 2 | |
|
3 | 3 | ========== |
|
4 | 4 | What's new |
|
5 | 5 | ========== |
|
6 | 6 | |
|
7 | 7 | .. contents:: |
|
8 | 8 | .. |
|
9 | 9 | 1 Release 0.9.1 |
|
10 | 10 | 2 Release 0.9 |
|
11 | 11 | 2.1 New features |
|
12 | 12 | 2.2 Bug fixes |
|
13 | 13 | 2.3 Backwards incompatible changes |
|
14 | 14 | 2.4 Changes merged in from IPython1 |
|
15 | 15 | 2.4.1 New features |
|
16 | 16 | 2.4.2 Bug fixes |
|
17 | 17 | 2.4.3 Backwards incompatible changes |
|
18 | 18 | 3 Release 0.8.4 |
|
19 | 19 | 4 Release 0.8.3 |
|
20 | 20 | 5 Release 0.8.2 |
|
21 | 21 | 6 Older releases |
|
22 | 22 | .. |
|
23 | 23 | |
|
24 | 24 | Release dev |
|
25 | 25 | =========== |
|
26 | 26 | |
|
27 | 27 | New features |
|
28 | 28 | ------------ |
|
29 | 29 | |
|
30 | 30 | * The new ipcluster now has a fully working ssh mode that should work on |
|
31 | 31 | Linux, Unix and OS X. Thanks to Vishal Vatsa for implementing this! |
|
32 | 32 | |
|
33 | 33 | * The wonderful TextMate editor can now be used with %edit on OS X. Thanks |
|
34 | 34 | to Matt Foster for this patch. |
|
35 | 35 | |
|
36 | 36 | * Fully refactored :command:`ipcluster` command line program for starting |
|
37 | 37 | IPython clusters. This new version is a complete rewrite and 1) is fully |
|
38 | 38 | cross platform (we now use Twisted's process management), 2) has much |
|
39 | 39 | improved performance, 3) uses subcommands for different types of clusters, |
|
40 | 40 | 4) uses argparse for parsing command line options, 5) has better support |
|
41 | 41 | for starting clusters using :command:`mpirun`, 6) has experimental support |
|
42 | 42 | for starting engines using PBS. However, this new version of ipcluster |
|
43 | 43 | should be considered a technology preview. We plan on changing the API |
|
44 | 44 | in significant ways before it is final. |
|
45 | 45 | |
|
46 | 46 | * The :mod:`argparse` module has been added to :mod:`IPython.external`. |
|
47 | 47 | |
|
48 | 48 | * Fully description of the security model added to the docs. |
|
49 | 49 | |
|
50 | 50 | * cd completer: show bookmarks if no other completions are available. |
|
51 | 51 | |
|
52 | 52 | * sh profile: easy way to give 'title' to prompt: assign to variable |
|
53 | 53 | '_prompt_title'. It looks like this:: |
|
54 | 54 | |
|
55 | 55 | [~]|1> _prompt_title = 'sudo!' |
|
56 | 56 | sudo![~]|2> |
|
57 | 57 | |
|
58 | 58 | * %edit: If you do '%edit pasted_block', pasted_block |
|
59 | 59 | variable gets updated with new data (so repeated |
|
60 | 60 | editing makes sense) |
|
61 | 61 | |
|
62 | 62 | Bug fixes |
|
63 | 63 | --------- |
|
64 | 64 | |
|
65 | 65 | * Numerous bugs on Windows with the new ipcluster have been fixed. |
|
66 | 66 | |
|
67 | 67 | * The ipengine and ipcontroller scripts now handle missing furl files |
|
68 | 68 | more gracefully by giving better error messages. |
|
69 | 69 | |
|
70 | 70 | * %rehashx: Aliases no longer contain dots. python3.0 binary |
|
71 | 71 | will create alias python30. Fixes: |
|
72 | 72 | #259716 "commands with dots in them don't work" |
|
73 | 73 | |
|
74 | 74 | * %cpaste: %cpaste -r repeats the last pasted block. |
|
75 | 75 | The block is assigned to pasted_block even if code |
|
76 | 76 | raises exception. |
|
77 | 77 | |
|
78 | * Bug #274067 'The code in get_home_dir is broken for py2exe' was | |
|
79 | fixed. | |
|
80 | ||
|
78 | 81 | Backwards incompatible changes |
|
79 | 82 | ------------------------------ |
|
80 | 83 | |
|
81 | 84 | * The controller now has a ``-r`` flag that needs to be used if you want to |
|
82 | 85 | reuse existing furl files. Otherwise they are deleted (the default). |
|
83 | 86 | |
|
84 | 87 | * Remove ipy_leo.py. "easy_install ipython-extension" to get it. |
|
85 | 88 | (done to decouple it from ipython release cycle) |
|
86 | 89 | |
|
87 | 90 | |
|
88 | 91 | |
|
89 | 92 | Release 0.9.1 |
|
90 | 93 | ============= |
|
91 | 94 | |
|
92 | 95 | This release was quickly made to restore compatibility with Python 2.4, which |
|
93 | 96 | version 0.9 accidentally broke. No new features were introduced, other than |
|
94 | 97 | some additional testing support for internal use. |
|
95 | 98 | |
|
96 | 99 | |
|
97 | 100 | Release 0.9 |
|
98 | 101 | =========== |
|
99 | 102 | |
|
100 | 103 | New features |
|
101 | 104 | ------------ |
|
102 | 105 | |
|
103 | 106 | * All furl files and security certificates are now put in a read-only |
|
104 | 107 | directory named ~./ipython/security. |
|
105 | 108 | |
|
106 | 109 | * A single function :func:`get_ipython_dir`, in :mod:`IPython.genutils` that |
|
107 | 110 | determines the user's IPython directory in a robust manner. |
|
108 | 111 | |
|
109 | 112 | * Laurent's WX application has been given a top-level script called |
|
110 | 113 | ipython-wx, and it has received numerous fixes. We expect this code to be |
|
111 | 114 | architecturally better integrated with Gael's WX 'ipython widget' over the |
|
112 | 115 | next few releases. |
|
113 | 116 | |
|
114 | 117 | * The Editor synchronization work by Vivian De Smedt has been merged in. This |
|
115 | 118 | code adds a number of new editor hooks to synchronize with editors under |
|
116 | 119 | Windows. |
|
117 | 120 | |
|
118 | 121 | * A new, still experimental but highly functional, WX shell by Gael Varoquaux. |
|
119 | 122 | This work was sponsored by Enthought, and while it's still very new, it is |
|
120 | 123 | based on a more cleanly organized arhictecture of the various IPython |
|
121 | 124 | components. We will continue to develop this over the next few releases as a |
|
122 | 125 | model for GUI components that use IPython. |
|
123 | 126 | |
|
124 | 127 | * Another GUI frontend, Cocoa based (Cocoa is the OSX native GUI framework), |
|
125 | 128 | authored by Barry Wark. Currently the WX and the Cocoa ones have slightly |
|
126 | 129 | different internal organizations, but the whole team is working on finding |
|
127 | 130 | what the right abstraction points are for a unified codebase. |
|
128 | 131 | |
|
129 | 132 | * As part of the frontend work, Barry Wark also implemented an experimental |
|
130 | 133 | event notification system that various ipython components can use. In the |
|
131 | 134 | next release the implications and use patterns of this system regarding the |
|
132 | 135 | various GUI options will be worked out. |
|
133 | 136 | |
|
134 | 137 | * IPython finally has a full test system, that can test docstrings with |
|
135 | 138 | IPython-specific functionality. There are still a few pieces missing for it |
|
136 | 139 | to be widely accessible to all users (so they can run the test suite at any |
|
137 | 140 | time and report problems), but it now works for the developers. We are |
|
138 | 141 | working hard on continuing to improve it, as this was probably IPython's |
|
139 | 142 | major Achilles heel (the lack of proper test coverage made it effectively |
|
140 | 143 | impossible to do large-scale refactoring). The full test suite can now |
|
141 | 144 | be run using the :command:`iptest` command line program. |
|
142 | 145 | |
|
143 | 146 | * The notion of a task has been completely reworked. An `ITask` interface has |
|
144 | 147 | been created. This interface defines the methods that tasks need to |
|
145 | 148 | implement. These methods are now responsible for things like submitting |
|
146 | 149 | tasks and processing results. There are two basic task types: |
|
147 | 150 | :class:`IPython.kernel.task.StringTask` (this is the old `Task` object, but |
|
148 | 151 | renamed) and the new :class:`IPython.kernel.task.MapTask`, which is based on |
|
149 | 152 | a function. |
|
150 | 153 | |
|
151 | 154 | * A new interface, :class:`IPython.kernel.mapper.IMapper` has been defined to |
|
152 | 155 | standardize the idea of a `map` method. This interface has a single `map` |
|
153 | 156 | method that has the same syntax as the built-in `map`. We have also defined |
|
154 | 157 | a `mapper` factory interface that creates objects that implement |
|
155 | 158 | :class:`IPython.kernel.mapper.IMapper` for different controllers. Both the |
|
156 | 159 | multiengine and task controller now have mapping capabilties. |
|
157 | 160 | |
|
158 | 161 | * The parallel function capabilities have been reworks. The major changes are |
|
159 | 162 | that i) there is now an `@parallel` magic that creates parallel functions, |
|
160 | 163 | ii) the syntax for mulitple variable follows that of `map`, iii) both the |
|
161 | 164 | multiengine and task controller now have a parallel function implementation. |
|
162 | 165 | |
|
163 | 166 | * All of the parallel computing capabilities from `ipython1-dev` have been |
|
164 | 167 | merged into IPython proper. This resulted in the following new subpackages: |
|
165 | 168 | :mod:`IPython.kernel`, :mod:`IPython.kernel.core`, :mod:`IPython.config`, |
|
166 | 169 | :mod:`IPython.tools` and :mod:`IPython.testing`. |
|
167 | 170 | |
|
168 | 171 | * As part of merging in the `ipython1-dev` stuff, the `setup.py` script and |
|
169 | 172 | friends have been completely refactored. Now we are checking for |
|
170 | 173 | dependencies using the approach that matplotlib uses. |
|
171 | 174 | |
|
172 | 175 | * The documentation has been completely reorganized to accept the |
|
173 | 176 | documentation from `ipython1-dev`. |
|
174 | 177 | |
|
175 | 178 | * We have switched to using Foolscap for all of our network protocols in |
|
176 | 179 | :mod:`IPython.kernel`. This gives us secure connections that are both |
|
177 | 180 | encrypted and authenticated. |
|
178 | 181 | |
|
179 | 182 | * We have a brand new `COPYING.txt` files that describes the IPython license |
|
180 | 183 | and copyright. The biggest change is that we are putting "The IPython |
|
181 | 184 | Development Team" as the copyright holder. We give more details about |
|
182 | 185 | exactly what this means in this file. All developer should read this and use |
|
183 | 186 | the new banner in all IPython source code files. |
|
184 | 187 | |
|
185 | 188 | * sh profile: ./foo runs foo as system command, no need to do !./foo anymore |
|
186 | 189 | |
|
187 | 190 | * String lists now support ``sort(field, nums = True)`` method (to easily sort |
|
188 | 191 | system command output). Try it with ``a = !ls -l ; a.sort(1, nums=1)``. |
|
189 | 192 | |
|
190 | 193 | * '%cpaste foo' now assigns the pasted block as string list, instead of string |
|
191 | 194 | |
|
192 | 195 | * The ipcluster script now run by default with no security. This is done |
|
193 | 196 | because the main usage of the script is for starting things on localhost. |
|
194 | 197 | Eventually when ipcluster is able to start things on other hosts, we will put |
|
195 | 198 | security back. |
|
196 | 199 | |
|
197 | 200 | * 'cd --foo' searches directory history for string foo, and jumps to that dir. |
|
198 | 201 | Last part of dir name is checked first. If no matches for that are found, |
|
199 | 202 | look at the whole path. |
|
200 | 203 | |
|
201 | 204 | |
|
202 | 205 | Bug fixes |
|
203 | 206 | --------- |
|
204 | 207 | |
|
205 | 208 | * The Windows installer has been fixed. Now all IPython scripts have ``.bat`` |
|
206 | 209 | versions created. Also, the Start Menu shortcuts have been updated. |
|
207 | 210 | |
|
208 | 211 | * The colors escapes in the multiengine client are now turned off on win32 as |
|
209 | 212 | they don't print correctly. |
|
210 | 213 | |
|
211 | 214 | * The :mod:`IPython.kernel.scripts.ipengine` script was exec'ing |
|
212 | 215 | mpi_import_statement incorrectly, which was leading the engine to crash when |
|
213 | 216 | mpi was enabled. |
|
214 | 217 | |
|
215 | 218 | * A few subpackages had missing ``__init__.py`` files. |
|
216 | 219 | |
|
217 | 220 | * The documentation is only created if Sphinx is found. Previously, the |
|
218 | 221 | ``setup.py`` script would fail if it was missing. |
|
219 | 222 | |
|
220 | 223 | * Greedy ``cd`` completion has been disabled again (it was enabled in 0.8.4) as |
|
221 | 224 | it caused problems on certain platforms. |
|
222 | 225 | |
|
223 | 226 | |
|
224 | 227 | Backwards incompatible changes |
|
225 | 228 | ------------------------------ |
|
226 | 229 | |
|
227 | 230 | * The ``clusterfile`` options of the :command:`ipcluster` command has been |
|
228 | 231 | removed as it was not working and it will be replaced soon by something much |
|
229 | 232 | more robust. |
|
230 | 233 | |
|
231 | 234 | * The :mod:`IPython.kernel` configuration now properly find the user's |
|
232 | 235 | IPython directory. |
|
233 | 236 | |
|
234 | 237 | * In ipapi, the :func:`make_user_ns` function has been replaced with |
|
235 | 238 | :func:`make_user_namespaces`, to support dict subclasses in namespace |
|
236 | 239 | creation. |
|
237 | 240 | |
|
238 | 241 | * :class:`IPython.kernel.client.Task` has been renamed |
|
239 | 242 | :class:`IPython.kernel.client.StringTask` to make way for new task types. |
|
240 | 243 | |
|
241 | 244 | * The keyword argument `style` has been renamed `dist` in `scatter`, `gather` |
|
242 | 245 | and `map`. |
|
243 | 246 | |
|
244 | 247 | * Renamed the values that the rename `dist` keyword argument can have from |
|
245 | 248 | `'basic'` to `'b'`. |
|
246 | 249 | |
|
247 | 250 | * IPython has a larger set of dependencies if you want all of its capabilities. |
|
248 | 251 | See the `setup.py` script for details. |
|
249 | 252 | |
|
250 | 253 | * The constructors for :class:`IPython.kernel.client.MultiEngineClient` and |
|
251 | 254 | :class:`IPython.kernel.client.TaskClient` no longer take the (ip,port) tuple. |
|
252 | 255 | Instead they take the filename of a file that contains the FURL for that |
|
253 | 256 | client. If the FURL file is in your IPYTHONDIR, it will be found automatically |
|
254 | 257 | and the constructor can be left empty. |
|
255 | 258 | |
|
256 | 259 | * The asynchronous clients in :mod:`IPython.kernel.asyncclient` are now created |
|
257 | 260 | using the factory functions :func:`get_multiengine_client` and |
|
258 | 261 | :func:`get_task_client`. These return a `Deferred` to the actual client. |
|
259 | 262 | |
|
260 | 263 | * The command line options to `ipcontroller` and `ipengine` have changed to |
|
261 | 264 | reflect the new Foolscap network protocol and the FURL files. Please see the |
|
262 | 265 | help for these scripts for details. |
|
263 | 266 | |
|
264 | 267 | * The configuration files for the kernel have changed because of the Foolscap |
|
265 | 268 | stuff. If you were using custom config files before, you should delete them |
|
266 | 269 | and regenerate new ones. |
|
267 | 270 | |
|
268 | 271 | Changes merged in from IPython1 |
|
269 | 272 | ------------------------------- |
|
270 | 273 | |
|
271 | 274 | New features |
|
272 | 275 | ............ |
|
273 | 276 | |
|
274 | 277 | * Much improved ``setup.py`` and ``setupegg.py`` scripts. Because Twisted and |
|
275 | 278 | zope.interface are now easy installable, we can declare them as dependencies |
|
276 | 279 | in our setupegg.py script. |
|
277 | 280 | |
|
278 | 281 | * IPython is now compatible with Twisted 2.5.0 and 8.x. |
|
279 | 282 | |
|
280 | 283 | * Added a new example of how to use :mod:`ipython1.kernel.asynclient`. |
|
281 | 284 | |
|
282 | 285 | * Initial draft of a process daemon in :mod:`ipython1.daemon`. This has not |
|
283 | 286 | been merged into IPython and is still in `ipython1-dev`. |
|
284 | 287 | |
|
285 | 288 | * The ``TaskController`` now has methods for getting the queue status. |
|
286 | 289 | |
|
287 | 290 | * The ``TaskResult`` objects not have information about how long the task |
|
288 | 291 | took to run. |
|
289 | 292 | |
|
290 | 293 | * We are attaching additional attributes to exceptions ``(_ipython_*)`` that |
|
291 | 294 | we use to carry additional info around. |
|
292 | 295 | |
|
293 | 296 | * New top-level module :mod:`asyncclient` that has asynchronous versions (that |
|
294 | 297 | return deferreds) of the client classes. This is designed to users who want |
|
295 | 298 | to run their own Twisted reactor. |
|
296 | 299 | |
|
297 | 300 | * All the clients in :mod:`client` are now based on Twisted. This is done by |
|
298 | 301 | running the Twisted reactor in a separate thread and using the |
|
299 | 302 | :func:`blockingCallFromThread` function that is in recent versions of Twisted. |
|
300 | 303 | |
|
301 | 304 | * Functions can now be pushed/pulled to/from engines using |
|
302 | 305 | :meth:`MultiEngineClient.push_function` and |
|
303 | 306 | :meth:`MultiEngineClient.pull_function`. |
|
304 | 307 | |
|
305 | 308 | * Gather/scatter are now implemented in the client to reduce the work load |
|
306 | 309 | of the controller and improve performance. |
|
307 | 310 | |
|
308 | 311 | * Complete rewrite of the IPython docuementation. All of the documentation |
|
309 | 312 | from the IPython website has been moved into docs/source as restructured |
|
310 | 313 | text documents. PDF and HTML documentation are being generated using |
|
311 | 314 | Sphinx. |
|
312 | 315 | |
|
313 | 316 | * New developer oriented documentation: development guidelines and roadmap. |
|
314 | 317 | |
|
315 | 318 | * Traditional ``ChangeLog`` has been changed to a more useful ``changes.txt`` |
|
316 | 319 | file that is organized by release and is meant to provide something more |
|
317 | 320 | relevant for users. |
|
318 | 321 | |
|
319 | 322 | Bug fixes |
|
320 | 323 | ......... |
|
321 | 324 | |
|
322 | 325 | * Created a proper ``MANIFEST.in`` file to create source distributions. |
|
323 | 326 | |
|
324 | 327 | * Fixed a bug in the ``MultiEngine`` interface. Previously, multi-engine |
|
325 | 328 | actions were being collected with a :class:`DeferredList` with |
|
326 | 329 | ``fireononeerrback=1``. This meant that methods were returning |
|
327 | 330 | before all engines had given their results. This was causing extremely odd |
|
328 | 331 | bugs in certain cases. To fix this problem, we have 1) set |
|
329 | 332 | ``fireononeerrback=0`` to make sure all results (or exceptions) are in |
|
330 | 333 | before returning and 2) introduced a :exc:`CompositeError` exception |
|
331 | 334 | that wraps all of the engine exceptions. This is a huge change as it means |
|
332 | 335 | that users will have to catch :exc:`CompositeError` rather than the actual |
|
333 | 336 | exception. |
|
334 | 337 | |
|
335 | 338 | Backwards incompatible changes |
|
336 | 339 | .............................. |
|
337 | 340 | |
|
338 | 341 | * All names have been renamed to conform to the lowercase_with_underscore |
|
339 | 342 | convention. This will require users to change references to all names like |
|
340 | 343 | ``queueStatus`` to ``queue_status``. |
|
341 | 344 | |
|
342 | 345 | * Previously, methods like :meth:`MultiEngineClient.push` and |
|
343 | 346 | :meth:`MultiEngineClient.push` used ``*args`` and ``**kwargs``. This was |
|
344 | 347 | becoming a problem as we weren't able to introduce new keyword arguments into |
|
345 | 348 | the API. Now these methods simple take a dict or sequence. This has also |
|
346 | 349 | allowed us to get rid of the ``*All`` methods like :meth:`pushAll` and |
|
347 | 350 | :meth:`pullAll`. These things are now handled with the ``targets`` keyword |
|
348 | 351 | argument that defaults to ``'all'``. |
|
349 | 352 | |
|
350 | 353 | * The :attr:`MultiEngineClient.magicTargets` has been renamed to |
|
351 | 354 | :attr:`MultiEngineClient.targets`. |
|
352 | 355 | |
|
353 | 356 | * All methods in the MultiEngine interface now accept the optional keyword |
|
354 | 357 | argument ``block``. |
|
355 | 358 | |
|
356 | 359 | * Renamed :class:`RemoteController` to :class:`MultiEngineClient` and |
|
357 | 360 | :class:`TaskController` to :class:`TaskClient`. |
|
358 | 361 | |
|
359 | 362 | * Renamed the top-level module from :mod:`api` to :mod:`client`. |
|
360 | 363 | |
|
361 | 364 | * Most methods in the multiengine interface now raise a :exc:`CompositeError` |
|
362 | 365 | exception that wraps the user's exceptions, rather than just raising the raw |
|
363 | 366 | user's exception. |
|
364 | 367 | |
|
365 | 368 | * Changed the ``setupNS`` and ``resultNames`` in the ``Task`` class to ``push`` |
|
366 | 369 | and ``pull``. |
|
367 | 370 | |
|
368 | 371 | |
|
369 | 372 | Release 0.8.4 |
|
370 | 373 | ============= |
|
371 | 374 | |
|
372 | 375 | This was a quick release to fix an unfortunate bug that slipped into the 0.8.3 |
|
373 | 376 | release. The ``--twisted`` option was disabled, as it turned out to be broken |
|
374 | 377 | across several platforms. |
|
375 | 378 | |
|
376 | 379 | |
|
377 | 380 | Release 0.8.3 |
|
378 | 381 | ============= |
|
379 | 382 | |
|
380 | 383 | * pydb is now disabled by default (due to %run -d problems). You can enable |
|
381 | 384 | it by passing -pydb command line argument to IPython. Note that setting |
|
382 | 385 | it in config file won't work. |
|
383 | 386 | |
|
384 | 387 | |
|
385 | 388 | Release 0.8.2 |
|
386 | 389 | ============= |
|
387 | 390 | |
|
388 | 391 | * %pushd/%popd behave differently; now "pushd /foo" pushes CURRENT directory |
|
389 | 392 | and jumps to /foo. The current behaviour is closer to the documented |
|
390 | 393 | behaviour, and should not trip anyone. |
|
391 | 394 | |
|
392 | 395 | |
|
393 | 396 | Older releases |
|
394 | 397 | ============== |
|
395 | 398 | |
|
396 | 399 | Changes in earlier releases of IPython are described in the older file |
|
397 | 400 | ``ChangeLog``. Please refer to this document for details. |
|
398 | 401 |
@@ -1,454 +1,485 b'' | |||
|
1 | 1 | .. _development: |
|
2 | 2 | |
|
3 | 3 | ============================== |
|
4 | 4 | IPython development guidelines |
|
5 | 5 | ============================== |
|
6 | 6 | |
|
7 | 7 | |
|
8 | 8 | Overview |
|
9 | 9 | ======== |
|
10 | 10 | |
|
11 | 11 | This document describes IPython from the perspective of developers. Most |
|
12 | 12 | importantly, it gives information for people who want to contribute to the |
|
13 | 13 | development of IPython. So if you want to help out, read on! |
|
14 | 14 | |
|
15 | 15 | How to contribute to IPython |
|
16 | 16 | ============================ |
|
17 | 17 | |
|
18 | 18 | IPython development is done using Bazaar [Bazaar]_ and Launchpad [Launchpad]_. |
|
19 | 19 | This makes it easy for people to contribute to the development of IPython. |
|
20 | 20 | There are several ways in which you can join in. |
|
21 | 21 | |
|
22 | 22 | If you have a small change that you want to send to the team, you can edit your |
|
23 | 23 | bazaar checkout of IPython (see below) in-place, and ask bazaar for the |
|
24 | 24 | differences:: |
|
25 | 25 | |
|
26 | 26 | $ cd /path/to/your/copy/of/ipython |
|
27 | 27 | $ bzr diff > my_fixes.diff |
|
28 | 28 | |
|
29 | 29 | This produces a patch file with your fixes, which we can apply to the source |
|
30 | 30 | tree. This file should then be attached to a ticket in our `bug tracker |
|
31 | 31 | <https://bugs.launchpad.net/ipython>`_, indicating what it does. |
|
32 | 32 | |
|
33 | 33 | This model of creating small, self-contained patches works very well and there |
|
34 | 34 | are open source projects that do their entire development this way. However, |
|
35 | 35 | in IPython we have found that for tracking larger changes, making use of |
|
36 | 36 | bazaar's full capabilities in conjunction with Launchpad's code hosting |
|
37 | 37 | services makes for a much better experience. |
|
38 | 38 | |
|
39 | 39 | Making your own branch of IPython allows you to refine your changes over time, |
|
40 | 40 | track the development of the main team, and propose your own full version of |
|
41 | 41 | the code for others to use and review, with a minimum amount of fuss. The next |
|
42 | 42 | parts of this document will explain how to do this. |
|
43 | 43 | |
|
44 | 44 | Install Bazaar and create a Launchpad account |
|
45 | 45 | --------------------------------------------- |
|
46 | 46 | |
|
47 | 47 | First make sure you have installed Bazaar (see their `website |
|
48 | 48 | <http://bazaar-vcs.org/>`_). To see that Bazaar is installed and knows about |
|
49 | 49 | you, try the following:: |
|
50 | 50 | |
|
51 | 51 | $ bzr whoami |
|
52 | 52 | Joe Coder <jcoder@gmail.com> |
|
53 | 53 | |
|
54 | 54 | This should display your name and email. Next, you will want to create an |
|
55 | 55 | account on the `Launchpad website <http://www.launchpad.net>`_ and setup your |
|
56 | 56 | ssh keys. For more information of setting up your ssh keys, see `this link |
|
57 | 57 | <https://help.launchpad.net/YourAccount/CreatingAnSSHKeyPair>`_. |
|
58 | 58 | |
|
59 | 59 | Get the main IPython branch from Launchpad |
|
60 | 60 | ------------------------------------------ |
|
61 | 61 | |
|
62 | 62 | Now, you can get a copy of the main IPython development branch (we call this |
|
63 | 63 | the "trunk"):: |
|
64 | 64 | |
|
65 | 65 | $ bzr branch lp:ipython |
|
66 | 66 | |
|
67 | 67 | Create a working branch |
|
68 | 68 | ----------------------- |
|
69 | 69 | |
|
70 | 70 | When working on IPython, you won't actually make edits directly to the |
|
71 | 71 | :file:`lp:ipython` branch. Instead, you will create a separate branch for your |
|
72 | 72 | changes. For now, let's assume you want to do your work in a branch named |
|
73 | 73 | "ipython-mybranch". Create this branch by doing:: |
|
74 | 74 | |
|
75 | 75 | $ bzr branch ipython ipython-mybranch |
|
76 | 76 | |
|
77 | 77 | When you actually create a branch, you will want to give it a name that |
|
78 | 78 | reflects the nature of the work that you will be doing in it, like |
|
79 | 79 | "install-docs-update". |
|
80 | 80 | |
|
81 | 81 | Make edits in your working branch |
|
82 | 82 | --------------------------------- |
|
83 | 83 | |
|
84 | 84 | Now you are ready to actually make edits in your :file:`ipython-mybranch` |
|
85 | 85 | branch. Before doing this, it is helpful to install this branch so you can |
|
86 | 86 | test your changes as you work. This is easiest if you have setuptools |
|
87 | 87 | installed. Then, just do:: |
|
88 | 88 | |
|
89 | 89 | $ cd ipython-mybranch |
|
90 | 90 | $ python setupegg.py develop |
|
91 | 91 | |
|
92 | 92 | Now, make some changes. After a while, you will want to commit your changes. |
|
93 | 93 | This let's Bazaar know that you like the changes you have made and gives you |
|
94 | 94 | an opportunity to keep a nice record of what you have done. This looks like |
|
95 | 95 | this:: |
|
96 | 96 | |
|
97 | 97 | $ ...do work in ipython-mybranch... |
|
98 | 98 | $ bzr commit -m "the commit message goes here" |
|
99 | 99 | |
|
100 | 100 | Please note that since we now don't use an old-style linear ChangeLog (that |
|
101 | 101 | tends to cause problems with distributed version control systems), you should |
|
102 | 102 | ensure that your log messages are reasonably detailed. Use a docstring-like |
|
103 | 103 | approach in the commit messages (including the second line being left |
|
104 | 104 | *blank*):: |
|
105 | 105 | |
|
106 | 106 | Single line summary of changes being committed. |
|
107 | 107 | |
|
108 | 108 | * more details when warranted ... |
|
109 | 109 | * including crediting outside contributors if they sent the |
|
110 | 110 | code/bug/idea! |
|
111 | 111 | |
|
112 | 112 | As you work, you will repeat this edit/commit cycle many times. If you work on |
|
113 | 113 | your branch for a long time, you will also want to get the latest changes from |
|
114 | 114 | the :file:`lp:ipython` branch. This can be done with the following sequence of |
|
115 | 115 | commands:: |
|
116 | 116 | |
|
117 | 117 | $ ls |
|
118 | 118 | ipython |
|
119 | 119 | ipython-mybranch |
|
120 | 120 | |
|
121 | 121 | $ cd ipython |
|
122 | 122 | $ bzr pull |
|
123 | 123 | $ cd ../ipython-mybranch |
|
124 | 124 | $ bzr merge ../ipython |
|
125 | 125 | $ bzr commit -m "Merging changes from trunk" |
|
126 | 126 | |
|
127 | 127 | Along the way, you should also run the IPython test suite. You can do this |
|
128 | 128 | using the :command:`iptest` command (which is basically a customized version of |
|
129 | 129 | :command:`nosetests`):: |
|
130 | 130 | |
|
131 | 131 | $ cd |
|
132 | 132 | $ iptest |
|
133 | 133 | |
|
134 | 134 | The :command:`iptest` command will also pick up and run any tests you have |
|
135 | 135 | written. See :ref:`_devel_testing` for further details on the testing system. |
|
136 | 136 | |
|
137 | 137 | |
|
138 | 138 | Post your branch and request a code review |
|
139 | 139 | ------------------------------------------ |
|
140 | 140 | |
|
141 | 141 | Once you are done with your edits, you should post your branch on Launchpad so |
|
142 | 142 | that other IPython developers can review the changes and help you merge your |
|
143 | 143 | changes into the main development branch. To post your branch on Launchpad, |
|
144 | 144 | do:: |
|
145 | 145 | |
|
146 | 146 | $ cd ipython-mybranch |
|
147 | 147 | $ bzr push lp:~yourusername/ipython/ipython-mybranch |
|
148 | 148 | |
|
149 | 149 | Then, go to the `IPython Launchpad site <www.launchpad.net/ipython>`_, and you |
|
150 | 150 | should see your branch under the "Code" tab. If you click on your branch, you |
|
151 | 151 | can provide a short description of the branch as well as mark its status. Most |
|
152 | 152 | importantly, you should click the link that reads "Propose for merging into |
|
153 | 153 | another branch". What does this do? |
|
154 | 154 | |
|
155 | 155 | This let's the other IPython developers know that your branch is ready to be |
|
156 | 156 | reviewed and merged into the main development branch. During this review |
|
157 | 157 | process, other developers will give you feedback and help you get your code |
|
158 | 158 | ready to be merged. What types of things will we be looking for: |
|
159 | 159 | |
|
160 | 160 | * All code is documented. |
|
161 | 161 | * All code has tests. |
|
162 | 162 | * The entire IPython test suite passes. |
|
163 | 163 | |
|
164 | 164 | Once your changes have been reviewed and approved, someone will merge them |
|
165 | 165 | into the main development branch. |
|
166 | 166 | |
|
167 | 167 | Documentation |
|
168 | 168 | ============= |
|
169 | 169 | |
|
170 | 170 | Standalone documentation |
|
171 | 171 | ------------------------ |
|
172 | 172 | |
|
173 | 173 | All standalone documentation should be written in plain text (``.txt``) files |
|
174 | 174 | using reStructuredText [reStructuredText]_ for markup and formatting. All such |
|
175 | 175 | documentation should be placed in directory :file:`docs/source` of the IPython |
|
176 | 176 | source tree. The documentation in this location will serve as the main source |
|
177 | 177 | for IPython documentation and all existing documentation should be converted |
|
178 | 178 | to this format. |
|
179 | 179 | |
|
180 | 180 | To build the final documentation, we use Sphinx [Sphinx]_. Once you have |
|
181 | 181 | Sphinx installed, you can build the html docs yourself by doing:: |
|
182 | 182 | |
|
183 | 183 | $ cd ipython-mybranch/docs |
|
184 | 184 | $ make html |
|
185 | 185 | |
|
186 | 186 | Docstring format |
|
187 | 187 | ---------------- |
|
188 | 188 | |
|
189 | 189 | Good docstrings are very important. All new code should have docstrings that |
|
190 | 190 | are formatted using reStructuredText for markup and formatting, since it is |
|
191 | 191 | understood by a wide variety of tools. Details about using reStructuredText |
|
192 | 192 | for docstrings can be found `here |
|
193 | 193 | <http://epydoc.sourceforge.net/manual-othermarkup.html>`_. |
|
194 | 194 | |
|
195 | 195 | Additional PEPs of interest regarding documentation of code: |
|
196 | 196 | |
|
197 | 197 | * `Docstring Conventions <http://www.python.org/peps/pep-0257.html>`_ |
|
198 | 198 | * `Docstring Processing System Framework <http://www.python.org/peps/pep-0256.html>`_ |
|
199 | 199 | * `Docutils Design Specification <http://www.python.org/peps/pep-0258.html>`_ |
|
200 | 200 | |
|
201 | 201 | |
|
202 | 202 | Coding conventions |
|
203 | 203 | ================== |
|
204 | 204 | |
|
205 | 205 | General |
|
206 | 206 | ------- |
|
207 | 207 | |
|
208 | 208 | In general, we'll try to follow the standard Python style conventions as |
|
209 | 209 | described here: |
|
210 | 210 | |
|
211 | 211 | * `Style Guide for Python Code <http://www.python.org/peps/pep-0008.html>`_ |
|
212 | 212 | |
|
213 | 213 | |
|
214 | 214 | Other comments: |
|
215 | 215 | |
|
216 | 216 | * In a large file, top level classes and functions should be |
|
217 | 217 | separated by 2-3 lines to make it easier to separate them visually. |
|
218 | 218 | * Use 4 spaces for indentation. |
|
219 | 219 | * Keep the ordering of methods the same in classes that have the same |
|
220 | 220 | methods. This is particularly true for classes that implement an interface. |
|
221 | 221 | |
|
222 | 222 | Naming conventions |
|
223 | 223 | ------------------ |
|
224 | 224 | |
|
225 | 225 | In terms of naming conventions, we'll follow the guidelines from the `Style |
|
226 | 226 | Guide for Python Code`_. |
|
227 | 227 | |
|
228 | 228 | For all new IPython code (and much existing code is being refactored), we'll |
|
229 | 229 | use: |
|
230 | 230 | |
|
231 | 231 | * All ``lowercase`` module names. |
|
232 | 232 | |
|
233 | 233 | * ``CamelCase`` for class names. |
|
234 | 234 | |
|
235 | 235 | * ``lowercase_with_underscores`` for methods, functions, variables and |
|
236 | 236 | attributes. |
|
237 | 237 | |
|
238 | 238 | There are, however, some important exceptions to these rules. In some cases, |
|
239 | 239 | IPython code will interface with packages (Twisted, Wx, Qt) that use other |
|
240 | 240 | conventions. At some level this makes it impossible to adhere to our own |
|
241 | 241 | standards at all times. In particular, when subclassing classes that use other |
|
242 | 242 | naming conventions, you must follow their naming conventions. To deal with |
|
243 | 243 | cases like this, we propose the following policy: |
|
244 | 244 | |
|
245 | 245 | * If you are subclassing a class that uses different conventions, use its |
|
246 | 246 | naming conventions throughout your subclass. Thus, if you are creating a |
|
247 | 247 | Twisted Protocol class, used Twisted's |
|
248 | 248 | ``namingSchemeForMethodsAndAttributes.`` |
|
249 | 249 | |
|
250 | 250 | * All IPython's official interfaces should use our conventions. In some cases |
|
251 | 251 | this will mean that you need to provide shadow names (first implement |
|
252 | 252 | ``fooBar`` and then ``foo_bar = fooBar``). We want to avoid this at all |
|
253 | 253 | costs, but it will probably be necessary at times. But, please use this |
|
254 | 254 | sparingly! |
|
255 | 255 | |
|
256 | 256 | Implementation-specific *private* methods will use |
|
257 | 257 | ``_single_underscore_prefix``. Names with a leading double underscore will |
|
258 | 258 | *only* be used in special cases, as they makes subclassing difficult (such |
|
259 | 259 | names are not easily seen by child classes). |
|
260 | 260 | |
|
261 | 261 | Occasionally some run-in lowercase names are used, but mostly for very short |
|
262 | 262 | names or where we are implementing methods very similar to existing ones in a |
|
263 | 263 | base class (like ``runlines()`` where ``runsource()`` and ``runcode()`` had |
|
264 | 264 | established precedent). |
|
265 | 265 | |
|
266 | 266 | The old IPython codebase has a big mix of classes and modules prefixed with an |
|
267 | 267 | explicit ``IP``. In Python this is mostly unnecessary, redundant and frowned |
|
268 | 268 | upon, as namespaces offer cleaner prefixing. The only case where this approach |
|
269 | 269 | is justified is for classes which are expected to be imported into external |
|
270 | 270 | namespaces and a very generic name (like Shell) is too likely to clash with |
|
271 | 271 | something else. We'll need to revisit this issue as we clean up and refactor |
|
272 | 272 | the code, but in general we should remove as many unnecessary ``IP``/``ip`` |
|
273 | 273 | prefixes as possible. However, if a prefix seems absolutely necessary the more |
|
274 | 274 | specific ``IPY`` or ``ipy`` are preferred. |
|
275 | 275 | |
|
276 | 276 | .. _devel_testing: |
|
277 | 277 | |
|
278 | 278 | Testing system |
|
279 | 279 | ============== |
|
280 | 280 | |
|
281 | 281 | It is extremely important that all code contributed to IPython has tests. |
|
282 | 282 | Tests should be written as unittests, doctests or as entities that the Nose |
|
283 | 283 | [Nose]_ testing package will find. Regardless of how the tests are written, we |
|
284 | 284 | will use Nose for discovering and running the tests. Nose will be required to |
|
285 | 285 | run the IPython test suite, but will not be required to simply use IPython. |
|
286 | 286 | |
|
287 | 287 | Tests of Twisted using code need to follow two additional guidelines: |
|
288 | 288 | |
|
289 | 289 | 1. Twisted using tests should be written by subclassing the :class:`TestCase` |
|
290 | 290 | class that comes with :mod:`twisted.trial.unittest`. |
|
291 | 291 | |
|
292 | 292 | 2. All :class:`Deferred` instances that are created in the test must be |
|
293 | 293 | properly chained and the final one *must* be the return value of the test |
|
294 | 294 | method. |
|
295 | 295 | |
|
296 | 296 | When these two things are done, Nose will be able to run the tests and the |
|
297 | 297 | twisted reactor will be handled correctly. |
|
298 | 298 | |
|
299 | 299 | Each subpackage in IPython should have its own :file:`tests` directory that |
|
300 | 300 | contains all of the tests for that subpackage. This allows each subpackage to |
|
301 | 301 | be self-contained. A good convention to follow is to have a file named |
|
302 | 302 | :file:`test_foo.py` for each module :file:`foo.py` in the package. This makes |
|
303 | 303 | it easy to organize the tests, though like most conventions, it's OK to break |
|
304 | 304 | it if logic and common sense dictate otherwise. |
|
305 | 305 | |
|
306 | 306 | If a subpackage has any dependencies beyond the Python standard library, the |
|
307 | 307 | tests for that subpackage should be skipped if the dependencies are not |
|
308 | 308 | found. This is very important so users don't get tests failing simply because |
|
309 | 309 | they don't have dependencies. We ship a set of decorators in the |
|
310 | 310 | :mod:`IPython.testing` package to tag tests that may be platform-specific or |
|
311 | 311 | otherwise may have restrictions; if the existing ones don't fit your needs, add |
|
312 | 312 | a new decorator in that location so other tests can reuse it. |
|
313 | 313 | |
|
314 | 314 | To run the IPython test suite, use the :command:`iptest` command that is |
|
315 | 315 | installed with IPython (if you are using IPython in-place, without installing |
|
316 | 316 | it, you can find this script in the :file:`scripts` directory):: |
|
317 | 317 | |
|
318 | 318 | $ iptest |
|
319 | 319 | |
|
320 | 320 | This command runs Nose with the proper options and extensions. By default, |
|
321 | 321 | :command:`iptest` runs the entire IPython test suite (skipping tests that may |
|
322 | 322 | be platform-specific or which depend on tools you may not have). But you can |
|
323 | 323 | also use it to run only one specific test file, or a specific test function. |
|
324 | 324 | For example, this will run only the :file:`test_magic` file from the test |
|
325 | 325 | suite:: |
|
326 | 326 | |
|
327 | 327 | $ iptest IPython.tests.test_magic |
|
328 | 328 | ---------------------------------------------------------------------- |
|
329 | 329 | Ran 10 tests in 0.348s |
|
330 | 330 | |
|
331 | 331 | OK (SKIP=3) |
|
332 | 332 | Deleting object: second_pass |
|
333 | 333 | |
|
334 | 334 | while the ``path:function`` syntax allows you to select a specific function in |
|
335 | 335 | that file to run:: |
|
336 | 336 | |
|
337 | 337 | $ iptest IPython.tests.test_magic:test_obj_del |
|
338 | 338 | ---------------------------------------------------------------------- |
|
339 | 339 | Ran 1 test in 0.204s |
|
340 | 340 | |
|
341 | 341 | OK |
|
342 | 342 | |
|
343 | 343 | Since :command:`iptest` is based on nosetests, you can pass it any regular |
|
344 | 344 | nosetests option. For example, you can use ``--pdb`` or ``--pdb-failures`` to |
|
345 | 345 | automatically activate the interactive Pdb debugger on errors or failures. See |
|
346 | 346 | the nosetests documentation for further details. |
|
347 | 347 | |
|
348 | .. warning:: | |
|
349 | ||
|
350 | Note that right now we have a nasty interaction between ipdoctest and | |
|
351 | twisted. Until we figure this out, please use the following instructions to | |
|
352 | ensure that at least you run all the tests. | |
|
353 | ||
|
354 | Right now, if you now run:: | |
|
355 | ||
|
356 | $ iptest [any options] [any submodules] | |
|
357 | ||
|
358 | it will NOT load ipdoctest but won't cause any Twisted problems. | |
|
359 | ||
|
360 | Once you're happy that you didn't break Twisted, run:: | |
|
361 | ||
|
362 | $ iptest --with-ipdoctest [any options] [any submodules] | |
|
363 | ||
|
364 | This MAY give a Twisted AlreadyCalledError exception at the end, but it will | |
|
365 | also correctly load up all of the ipython-specific tests and doctests. | |
|
366 | ||
|
367 | The above can be made easier with a trivial shell alias:: | |
|
368 | ||
|
369 | $ alias iptest2='iptest --with-ipdoctest' | |
|
370 | ||
|
371 | So that you can run:: | |
|
372 | ||
|
373 | $ iptest ... | |
|
374 | # Twisted happy | |
|
375 | # iptest2 ... | |
|
376 | # ignore possible Twisted error, this checks all the rest. | |
|
377 | ||
|
378 | ||
|
348 | 379 | A few tips for writing tests |
|
349 | 380 | ---------------------------- |
|
350 | 381 | |
|
351 | 382 | You can write tests either as normal test files, using all the conventions that |
|
352 | 383 | Nose recognizes, or as doctests. Note that *all* IPython functions should have |
|
353 | 384 | at least one example that serves as a doctest, whenever technically feasible. |
|
354 | 385 | However, example doctests should only be in the main docstring if they are *a |
|
355 | 386 | good example*, i.e. if they convey useful information about the function. If |
|
356 | 387 | you simply would like to write a test as a doctest, put it in a separate test |
|
357 | 388 | file and write a no-op function whose only purpose is its docstring. |
|
358 | 389 | |
|
359 | 390 | Note, however, that in a file named :file:`test_X`, functions whose only test |
|
360 | 391 | is their docstring (as a doctest) and which have no test functionality of their |
|
361 | 392 | own, should be called *doctest_foo* instead of *test_foo*, otherwise they get |
|
362 | 393 | double-counted (the empty function call is counted as a test, which just |
|
363 | 394 | inflates tests numbers artificially). This restriction does not apply to |
|
364 | 395 | functions in files with other names, due to how Nose discovers tests. |
|
365 | 396 | |
|
366 | 397 | You can use IPython examples in your docstrings. Those can make full use of |
|
367 | 398 | IPython functionality (magics, variable substitution, etc), but be careful to |
|
368 | 399 | keep them generic enough that they run identically on all Operating Systems. |
|
369 | 400 | |
|
370 | 401 | The prompts in your doctests can be either of the plain Python ``>>>`` variety |
|
371 | 402 | or ``In [1]:`` IPython style. Since this is the IPython system, after all, we |
|
372 | 403 | encourage you to use IPython prompts throughout, unless you are illustrating a |
|
373 | 404 | specific aspect of the normal prompts (such as the ``%doctest_mode`` magic). |
|
374 | 405 | |
|
375 | 406 | If a test isn't safe to run inside the main nose process (e.g. because it loads |
|
376 | 407 | a GUI toolkit), consider running it in a subprocess and capturing its output |
|
377 | 408 | for evaluation and test decision later. Here is an example of how to do it, by |
|
378 | 409 | relying on the builtin ``_ip`` object that contains the public IPython api as |
|
379 | 410 | defined in :mod:`IPython.ipapi`:: |
|
380 | 411 | |
|
381 | 412 | def test_obj_del(): |
|
382 | 413 | """Test that object's __del__ methods are called on exit.""" |
|
383 | 414 | test_dir = os.path.dirname(__file__) |
|
384 | 415 | del_file = os.path.join(test_dir,'obj_del.py') |
|
385 | 416 | out = _ip.IP.getoutput('ipython %s' % del_file) |
|
386 | 417 | nt.assert_equals(out,'object A deleted') |
|
387 | 418 | |
|
388 | 419 | |
|
389 | 420 | |
|
390 | 421 | If a doctest contains input whose output you don't want to verify identically |
|
391 | 422 | via doctest (random output, an object id, etc), you can mark a docstring with |
|
392 | 423 | ``#random``. All of these test will have their code executed but no output |
|
393 | 424 | checking will be done:: |
|
394 | 425 | |
|
395 | 426 | >>> 1+3 |
|
396 | 427 | junk goes here... # random |
|
397 | 428 | |
|
398 | 429 | >>> 1+2 |
|
399 | 430 | again, anything goes #random |
|
400 | 431 | if multiline, the random mark is only needed once. |
|
401 | 432 | |
|
402 | 433 | >>> 1+2 |
|
403 | 434 | You can also put the random marker at the end: |
|
404 | 435 | # random |
|
405 | 436 | |
|
406 | 437 | >>> 1+2 |
|
407 | 438 | # random |
|
408 | 439 | .. or at the beginning. |
|
409 | 440 | |
|
410 | 441 | In a case where you want an *entire* docstring to be executed but not verified |
|
411 | 442 | (this only serves to check that the code runs without crashing, so it should be |
|
412 | 443 | used very sparingly), you can put ``# all-random`` in the docstring. |
|
413 | 444 | |
|
414 | 445 | .. _devel_config: |
|
415 | 446 | |
|
416 | 447 | Release checklist |
|
417 | 448 | ================= |
|
418 | 449 | |
|
419 | 450 | Most of the release process is automated by the :file:`release` script in the |
|
420 | 451 | :file:`tools` directory. This is just a handy reminder for the release manager. |
|
421 | 452 | |
|
422 | 453 | #. Run the release script, which makes the tar.gz, eggs and Win32 .exe |
|
423 | 454 | installer. It posts them to the site and registers the release with PyPI. |
|
424 | 455 | |
|
425 | 456 | #. Updating the website with announcements and links to the updated |
|
426 | 457 | changes.txt in html form. Remember to put a short note both on the news |
|
427 | 458 | page of the site and on Launcphad. |
|
428 | 459 | |
|
429 | 460 | #. Drafting a short release announcement with i) highlights and ii) a link to |
|
430 | 461 | the html changes.txt. |
|
431 | 462 | |
|
432 | 463 | #. Make sure that the released version of the docs is live on the site. |
|
433 | 464 | |
|
434 | 465 | #. Celebrate! |
|
435 | 466 | |
|
436 | 467 | Porting to 3.0 |
|
437 | 468 | ============== |
|
438 | 469 | |
|
439 | 470 | There are no definite plans for porting of IPython to python 3. The major |
|
440 | 471 | issue is the dependency on twisted framework for the networking/threading |
|
441 | 472 | stuff. It is possible that it the traditional IPython interactive console |
|
442 | 473 | could be ported more easily since it has no such dependency. Here are a few |
|
443 | 474 | things that will need to be considered when doing such a port especially |
|
444 | 475 | if we want to have a codebase that works directly on both 2.x and 3.x. |
|
445 | 476 | |
|
446 | 477 | 1. The syntax for exceptions changed (PEP 3110). The old |
|
447 | 478 | `except exc, var` changed to `except exc as var`. At last |
|
448 | 479 | count there was 78 occurences of this usage in the codebase |
|
449 | 480 | |
|
450 | 481 | .. [Bazaar] Bazaar. http://bazaar-vcs.org/ |
|
451 | 482 | .. [Launchpad] Launchpad. http://www.launchpad.net/ipython |
|
452 | 483 | .. [reStructuredText] reStructuredText. http://docutils.sourceforge.net/rst.html |
|
453 | 484 | .. [Sphinx] Sphinx. http://sphinx.pocoo.org/ |
|
454 | 485 | .. [Nose] Nose: a discovery based unittest extension. http://code.google.com/p/python-nose/ |
@@ -1,157 +1,157 b'' | |||
|
1 | 1 | .. _parallelmpi: |
|
2 | 2 | |
|
3 | 3 | ======================= |
|
4 | 4 | Using MPI with IPython |
|
5 | 5 | ======================= |
|
6 | 6 | |
|
7 | 7 | Often, a parallel algorithm will require moving data between the engines. One way of accomplishing this is by doing a pull and then a push using the multiengine client. However, this will be slow as all the data has to go through the controller to the client and then back through the controller, to its final destination. |
|
8 | 8 | |
|
9 | 9 | A much better way of moving data between engines is to use a message passing library, such as the Message Passing Interface (MPI) [MPI]_. IPython's parallel computing architecture has been designed from the ground up to integrate with MPI. This document describes how to use MPI with IPython. |
|
10 | 10 | |
|
11 | 11 | Additional installation requirements |
|
12 | 12 | ==================================== |
|
13 | 13 | |
|
14 | 14 | If you want to use MPI with IPython, you will need to install: |
|
15 | 15 | |
|
16 | 16 | * A standard MPI implementation such as OpenMPI [OpenMPI]_ or MPICH. |
|
17 | 17 | * The mpi4py [mpi4py]_ package. |
|
18 | 18 | |
|
19 | 19 | .. note:: |
|
20 | 20 | |
|
21 | 21 | The mpi4py package is not a strict requirement. However, you need to |
|
22 | 22 | have *some* way of calling MPI from Python. You also need some way of |
|
23 | 23 | making sure that :func:`MPI_Init` is called when the IPython engines start |
|
24 | 24 | up. There are a number of ways of doing this and a good number of |
|
25 | 25 | associated subtleties. We highly recommend just using mpi4py as it |
|
26 | 26 | takes care of most of these problems. If you want to do something |
|
27 | 27 | different, let us know and we can help you get started. |
|
28 | 28 | |
|
29 | 29 | Starting the engines with MPI enabled |
|
30 | 30 | ===================================== |
|
31 | 31 | |
|
32 | 32 | To use code that calls MPI, there are typically two things that MPI requires. |
|
33 | 33 | |
|
34 | 34 | 1. The process that wants to call MPI must be started using |
|
35 |
:command:`mpi |
|
|
35 | :command:`mpiexec` or a batch system (like PBS) that has MPI support. | |
|
36 | 36 | 2. Once the process starts, it must call :func:`MPI_Init`. |
|
37 | 37 | |
|
38 | 38 | There are a couple of ways that you can start the IPython engines and get these things to happen. |
|
39 | 39 | |
|
40 |
Automatic starting using :command:`mpi |
|
|
40 | Automatic starting using :command:`mpiexec` and :command:`ipcluster` | |
|
41 | 41 | ------------------------------------------------------------------- |
|
42 | 42 | |
|
43 |
The easiest approach is to use the `mpi |
|
|
43 | The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`, which will first start a controller and then a set of engines using :command:`mpiexec`:: | |
|
44 | 44 | |
|
45 |
$ ipcluster mpi |
|
|
45 | $ ipcluster mpiexec -n 4 | |
|
46 | 46 | |
|
47 | 47 | This approach is best as interrupting :command:`ipcluster` will automatically |
|
48 | 48 | stop and clean up the controller and engines. |
|
49 | 49 | |
|
50 |
Manual starting using :command:`mpi |
|
|
50 | Manual starting using :command:`mpiexec` | |
|
51 | 51 | --------------------------------------- |
|
52 | 52 | |
|
53 |
If you want to start the IPython engines using the :command:`mpi |
|
|
53 | If you want to start the IPython engines using the :command:`mpiexec`, just do:: | |
|
54 | 54 | |
|
55 |
$ mpi |
|
|
55 | $ mpiexec -n 4 ipengine --mpi=mpi4py | |
|
56 | 56 | |
|
57 | 57 | This requires that you already have a controller running and that the FURL |
|
58 | 58 | files for the engines are in place. We also have built in support for |
|
59 | 59 | PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by |
|
60 | 60 | starting the engines with:: |
|
61 | 61 | |
|
62 |
mpi |
|
|
62 | mpiexec -n 4 ipengine --mpi=pytrilinos | |
|
63 | 63 | |
|
64 | 64 | Automatic starting using PBS and :command:`ipcluster` |
|
65 | 65 | ----------------------------------------------------- |
|
66 | 66 | |
|
67 | 67 | The :command:`ipcluster` command also has built-in integration with PBS. For more information on this approach, see our documentation on :ref:`ipcluster <parallel_process>`. |
|
68 | 68 | |
|
69 | 69 | Actually using MPI |
|
70 | 70 | ================== |
|
71 | 71 | |
|
72 | 72 | Once the engines are running with MPI enabled, you are ready to go. You can now call any code that uses MPI in the IPython engines. And, all of this can be done interactively. Here we show a simple example that uses mpi4py [mpi4py]_. |
|
73 | 73 | |
|
74 | 74 | First, lets define a simply function that uses MPI to calculate the sum of a distributed array. Save the following text in a file called :file:`psum.py`: |
|
75 | 75 | |
|
76 | 76 | .. sourcecode:: python |
|
77 | 77 | |
|
78 | 78 | from mpi4py import MPI |
|
79 | 79 | import numpy as np |
|
80 | 80 | |
|
81 | 81 | def psum(a): |
|
82 | 82 | s = np.sum(a) |
|
83 | 83 | return MPI.COMM_WORLD.Allreduce(s,MPI.SUM) |
|
84 | 84 | |
|
85 | 85 | Now, start an IPython cluster in the same directory as :file:`psum.py`:: |
|
86 | 86 | |
|
87 |
$ ipcluster mpi |
|
|
87 | $ ipcluster mpiexec -n 4 | |
|
88 | 88 | |
|
89 | 89 | Finally, connect to the cluster and use this function interactively. In this case, we create a random array on each engine and sum up all the random arrays using our :func:`psum` function: |
|
90 | 90 | |
|
91 | 91 | .. sourcecode:: ipython |
|
92 | 92 | |
|
93 | 93 | In [1]: from IPython.kernel import client |
|
94 | 94 | |
|
95 | 95 | In [2]: mec = client.MultiEngineClient() |
|
96 | 96 | |
|
97 | 97 | In [3]: mec.activate() |
|
98 | 98 | |
|
99 | 99 | In [4]: px import numpy as np |
|
100 | 100 | Parallel execution on engines: all |
|
101 | 101 | Out[4]: |
|
102 | 102 | <Results List> |
|
103 | 103 | [0] In [13]: import numpy as np |
|
104 | 104 | [1] In [13]: import numpy as np |
|
105 | 105 | [2] In [13]: import numpy as np |
|
106 | 106 | [3] In [13]: import numpy as np |
|
107 | 107 | |
|
108 | 108 | In [6]: px a = np.random.rand(100) |
|
109 | 109 | Parallel execution on engines: all |
|
110 | 110 | Out[6]: |
|
111 | 111 | <Results List> |
|
112 | 112 | [0] In [15]: a = np.random.rand(100) |
|
113 | 113 | [1] In [15]: a = np.random.rand(100) |
|
114 | 114 | [2] In [15]: a = np.random.rand(100) |
|
115 | 115 | [3] In [15]: a = np.random.rand(100) |
|
116 | 116 | |
|
117 | 117 | In [7]: px from psum import psum |
|
118 | 118 | Parallel execution on engines: all |
|
119 | 119 | Out[7]: |
|
120 | 120 | <Results List> |
|
121 | 121 | [0] In [16]: from psum import psum |
|
122 | 122 | [1] In [16]: from psum import psum |
|
123 | 123 | [2] In [16]: from psum import psum |
|
124 | 124 | [3] In [16]: from psum import psum |
|
125 | 125 | |
|
126 | 126 | In [8]: px s = psum(a) |
|
127 | 127 | Parallel execution on engines: all |
|
128 | 128 | Out[8]: |
|
129 | 129 | <Results List> |
|
130 | 130 | [0] In [17]: s = psum(a) |
|
131 | 131 | [1] In [17]: s = psum(a) |
|
132 | 132 | [2] In [17]: s = psum(a) |
|
133 | 133 | [3] In [17]: s = psum(a) |
|
134 | 134 | |
|
135 | 135 | In [9]: px print s |
|
136 | 136 | Parallel execution on engines: all |
|
137 | 137 | Out[9]: |
|
138 | 138 | <Results List> |
|
139 | 139 | [0] In [18]: print s |
|
140 | 140 | [0] Out[18]: 187.451545803 |
|
141 | 141 | |
|
142 | 142 | [1] In [18]: print s |
|
143 | 143 | [1] Out[18]: 187.451545803 |
|
144 | 144 | |
|
145 | 145 | [2] In [18]: print s |
|
146 | 146 | [2] Out[18]: 187.451545803 |
|
147 | 147 | |
|
148 | 148 | [3] In [18]: print s |
|
149 | 149 | [3] Out[18]: 187.451545803 |
|
150 | 150 | |
|
151 | 151 | Any Python code that makes calls to MPI can be used in this manner, including |
|
152 | 152 | compiled C, C++ and Fortran libraries that have been exposed to Python. |
|
153 | 153 | |
|
154 | 154 | .. [MPI] Message Passing Interface. http://www-unix.mcs.anl.gov/mpi/ |
|
155 | 155 | .. [mpi4py] MPI for Python. mpi4py: http://mpi4py.scipy.org/ |
|
156 | 156 | .. [OpenMPI] Open MPI. http://www.open-mpi.org/ |
|
157 | 157 | .. [PyTrilinos] PyTrilinos. http://trilinos.sandia.gov/packages/pytrilinos/ No newline at end of file |
@@ -1,324 +1,336 b'' | |||
|
1 | 1 | .. _parallel_process: |
|
2 | 2 | |
|
3 | 3 | =========================================== |
|
4 | 4 | Starting the IPython controller and engines |
|
5 | 5 | =========================================== |
|
6 | 6 | |
|
7 | 7 | To use IPython for parallel computing, you need to start one instance of |
|
8 | 8 | the controller and one or more instances of the engine. The controller |
|
9 | 9 | and each engine can run on different machines or on the same machine. |
|
10 | 10 | Because of this, there are many different possibilities. |
|
11 | 11 | |
|
12 | 12 | Broadly speaking, there are two ways of going about starting a controller and engines: |
|
13 | 13 | |
|
14 | 14 | * In an automated manner using the :command:`ipcluster` command. |
|
15 | 15 | * In a more manual way using the :command:`ipcontroller` and |
|
16 | 16 | :command:`ipengine` commands. |
|
17 | 17 | |
|
18 | 18 | This document describes both of these methods. We recommend that new users start with the :command:`ipcluster` command as it simplifies many common usage cases. |
|
19 | 19 | |
|
20 | 20 | General considerations |
|
21 | 21 | ====================== |
|
22 | 22 | |
|
23 | 23 | Before delving into the details about how you can start a controller and engines using the various methods, we outline some of the general issues that come up when starting the controller and engines. These things come up no matter which method you use to start your IPython cluster. |
|
24 | 24 | |
|
25 | 25 | Let's say that you want to start the controller on ``host0`` and engines on hosts ``host1``-``hostn``. The following steps are then required: |
|
26 | 26 | |
|
27 | 27 | 1. Start the controller on ``host0`` by running :command:`ipcontroller` on |
|
28 | 28 | ``host0``. |
|
29 | 29 | 2. Move the FURL file (:file:`ipcontroller-engine.furl`) created by the |
|
30 | 30 | controller from ``host0`` to hosts ``host1``-``hostn``. |
|
31 | 31 | 3. Start the engines on hosts ``host1``-``hostn`` by running |
|
32 | 32 | :command:`ipengine`. This command has to be told where the FURL file |
|
33 | 33 | (:file:`ipcontroller-engine.furl`) is located. |
|
34 | 34 | |
|
35 | 35 | At this point, the controller and engines will be connected. By default, the |
|
36 | 36 | FURL files created by the controller are put into the |
|
37 | 37 | :file:`~/.ipython/security` directory. If the engines share a filesystem with |
|
38 | 38 | the controller, step 2 can be skipped as the engines will automatically look |
|
39 | 39 | at that location. |
|
40 | 40 | |
|
41 | 41 | The final step required required to actually use the running controller from a |
|
42 | 42 | client is to move the FURL files :file:`ipcontroller-mec.furl` and |
|
43 | 43 | :file:`ipcontroller-tc.furl` from ``host0`` to the host where the clients will |
|
44 | 44 | be run. If these file are put into the :file:`~/.ipython/security` directory of the client's host, they will be found automatically. Otherwise, the full path to them has to be passed to the client's constructor. |
|
45 | 45 | |
|
46 | 46 | Using :command:`ipcluster` |
|
47 | 47 | ========================== |
|
48 | 48 | |
|
49 | 49 | The :command:`ipcluster` command provides a simple way of starting a controller and engines in the following situations: |
|
50 | 50 | |
|
51 | 51 | 1. When the controller and engines are all run on localhost. This is useful |
|
52 | 52 | for testing or running on a multicore computer. |
|
53 | 53 | 2. When engines are started using the :command:`mpirun` command that comes |
|
54 | 54 | with most MPI [MPI]_ implementations |
|
55 | 55 | 3. When engines are started using the PBS [PBS]_ batch system. |
|
56 | 56 | 4. When the controller is started on localhost and the engines are started on |
|
57 | 57 | remote nodes using :command:`ssh`. |
|
58 | 58 | |
|
59 | 59 | .. note:: |
|
60 | 60 | |
|
61 | 61 | It is also possible for advanced users to add support to |
|
62 | 62 | :command:`ipcluster` for starting controllers and engines using other |
|
63 | 63 | methods (like Sun's Grid Engine for example). |
|
64 | 64 | |
|
65 | 65 | .. note:: |
|
66 | 66 | |
|
67 | 67 | Currently :command:`ipcluster` requires that the |
|
68 | 68 | :file:`~/.ipython/security` directory live on a shared filesystem that is |
|
69 | 69 | seen by both the controller and engines. If you don't have a shared file |
|
70 | 70 | system you will need to use :command:`ipcontroller` and |
|
71 | 71 | :command:`ipengine` directly. This constraint can be relaxed if you are |
|
72 | 72 | using the :command:`ssh` method to start the cluster. |
|
73 | 73 | |
|
74 | 74 | Underneath the hood, :command:`ipcluster` just uses :command:`ipcontroller` |
|
75 | 75 | and :command:`ipengine` to perform the steps described above. |
|
76 | 76 | |
|
77 | 77 | Using :command:`ipcluster` in local mode |
|
78 | 78 | ---------------------------------------- |
|
79 | 79 | |
|
80 | 80 | To start one controller and 4 engines on localhost, just do:: |
|
81 | 81 | |
|
82 | 82 | $ ipcluster local -n 4 |
|
83 | 83 | |
|
84 | 84 | To see other command line options for the local mode, do:: |
|
85 | 85 | |
|
86 | 86 | $ ipcluster local -h |
|
87 | 87 | |
|
88 | Using :command:`ipcluster` in mpirun mode | |
|
89 | ----------------------------------------- | |
|
88 | Using :command:`ipcluster` in mpiexec/mpirun mode | |
|
89 | ------------------------------------------------- | |
|
90 | 90 | |
|
91 | The mpirun mode is useful if you: | |
|
91 | The mpiexec/mpirun mode is useful if you: | |
|
92 | 92 | |
|
93 | 93 | 1. Have MPI installed. |
|
94 |
2. Your systems are configured to use the :command:`mpi |
|
|
95 | processes. | |
|
94 | 2. Your systems are configured to use the :command:`mpiexec` or | |
|
95 | :command:`mpirun` commands to start MPI processes. | |
|
96 | ||
|
97 | .. note:: | |
|
98 | ||
|
99 | The preferred command to use is :command:`mpiexec`. However, we also | |
|
100 | support :command:`mpirun` for backwards compatibility. The underlying | |
|
101 | logic used is exactly the same, the only difference being the name of the | |
|
102 | command line program that is called. | |
|
96 | 103 | |
|
97 | 104 | If these are satisfied, you can start an IPython cluster using:: |
|
98 | 105 | |
|
99 |
$ ipcluster mpi |
|
|
106 | $ ipcluster mpiexec -n 4 | |
|
100 | 107 | |
|
101 | 108 | This does the following: |
|
102 | 109 | |
|
103 | 110 | 1. Starts the IPython controller on current host. |
|
104 |
2. Uses :command:`mpi |
|
|
111 | 2. Uses :command:`mpiexec` to start 4 engines. | |
|
105 | 112 | |
|
106 | 113 | On newer MPI implementations (such as OpenMPI), this will work even if you don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI implementations actually require each process to call :func:`MPI_Init` upon starting. The easiest way of having this done is to install the mpi4py [mpi4py]_ package and then call ipcluster with the ``--mpi`` option:: |
|
107 | 114 | |
|
108 |
$ ipcluster mpi |
|
|
115 | $ ipcluster mpiexec -n 4 --mpi=mpi4py | |
|
109 | 116 | |
|
110 | 117 | Unfortunately, even this won't work for some MPI implementations. If you are having problems with this, you will likely have to use a custom Python executable that itself calls :func:`MPI_Init` at the appropriate time. Fortunately, mpi4py comes with such a custom Python executable that is easy to install and use. However, this custom Python executable approach will not work with :command:`ipcluster` currently. |
|
111 | 118 | |
|
112 | 119 | Additional command line options for this mode can be found by doing:: |
|
113 | 120 | |
|
114 |
$ ipcluster mpi |
|
|
121 | $ ipcluster mpiexec -h | |
|
115 | 122 | |
|
116 | 123 | More details on using MPI with IPython can be found :ref:`here <parallelmpi>`. |
|
117 | 124 | |
|
118 | 125 | |
|
119 | 126 | Using :command:`ipcluster` in PBS mode |
|
120 | 127 | -------------------------------------- |
|
121 | 128 | |
|
122 | 129 | The PBS mode uses the Portable Batch System [PBS]_ to start the engines. To use this mode, you first need to create a PBS script template that will be used to start the engines. Here is a sample PBS script template: |
|
123 | 130 | |
|
124 | 131 | .. sourcecode:: bash |
|
125 | 132 | |
|
126 | 133 | #PBS -N ipython |
|
127 | 134 | #PBS -j oe |
|
128 | 135 | #PBS -l walltime=00:10:00 |
|
129 | 136 | #PBS -l nodes=${n/4}:ppn=4 |
|
130 | 137 | #PBS -q parallel |
|
131 | 138 | |
|
132 | 139 | cd $$PBS_O_WORKDIR |
|
133 | 140 | export PATH=$$HOME/usr/local/bin |
|
134 | 141 | export PYTHONPATH=$$HOME/usr/local/lib/python2.4/site-packages |
|
135 | 142 | /usr/local/bin/mpiexec -n ${n} ipengine --logfile=$$PBS_O_WORKDIR/ipengine |
|
136 | 143 | |
|
137 | 144 | There are a few important points about this template: |
|
138 | 145 | |
|
139 | 146 | 1. This template will be rendered at runtime using IPython's :mod:`Itpl` |
|
140 | 147 | template engine. |
|
141 | 148 | |
|
142 | 149 | 2. Instead of putting in the actual number of engines, use the notation |
|
143 | 150 | ``${n}`` to indicate the number of engines to be started. You can also uses |
|
144 | 151 | expressions like ``${n/4}`` in the template to indicate the number of |
|
145 | 152 | nodes. |
|
146 | 153 | |
|
147 | 154 | 3. Because ``$`` is a special character used by the template engine, you must |
|
148 | 155 | escape any ``$`` by using ``$$``. This is important when referring to |
|
149 | 156 | environment variables in the template. |
|
150 | 157 | |
|
151 | 158 | 4. Any options to :command:`ipengine` should be given in the batch script |
|
152 | 159 | template. |
|
153 | 160 | |
|
154 | 161 | 5. Depending on the configuration of you system, you may have to set |
|
155 | 162 | environment variables in the script template. |
|
156 | 163 | |
|
157 | 164 | Once you have created such a script, save it with a name like :file:`pbs.template`. Now you are ready to start your job:: |
|
158 | 165 | |
|
159 | 166 | $ ipcluster pbs -n 128 --pbs-script=pbs.template |
|
160 | 167 | |
|
161 | 168 | Additional command line options for this mode can be found by doing:: |
|
162 | 169 | |
|
163 | 170 | $ ipcluster pbs -h |
|
164 | 171 | |
|
165 | 172 | Using :command:`ipcluster` in SSH mode |
|
166 | 173 | -------------------------------------- |
|
167 | 174 | |
|
168 | 175 | The SSH mode uses :command:`ssh` to execute :command:`ipengine` on remote |
|
169 | 176 | nodes and the :command:`ipcontroller` on localhost. |
|
170 | 177 | |
|
171 | 178 | When using using this mode it highly recommended that you have set up SSH keys and are using ssh-agent [SSH]_ for password-less logins. |
|
172 | 179 | |
|
173 | 180 | To use this mode you need a python file describing the cluster, here is an example of such a "clusterfile": |
|
174 | 181 | |
|
175 | 182 | .. sourcecode:: python |
|
176 | 183 | |
|
177 | 184 | send_furl = True |
|
178 | 185 | engines = { 'host1.example.com' : 2, |
|
179 | 186 | 'host2.example.com' : 5, |
|
180 | 187 | 'host3.example.com' : 1, |
|
181 | 188 | 'host4.example.com' : 8 } |
|
182 | 189 | |
|
183 | 190 | Since this is a regular python file usual python syntax applies. Things to note: |
|
184 | 191 | |
|
185 | 192 | * The `engines` dict, where the keys is the host we want to run engines on and |
|
186 | 193 | the value is the number of engines to run on that host. |
|
187 | 194 | * send_furl can either be `True` or `False`, if `True` it will copy over the |
|
188 | 195 | furl needed for :command:`ipengine` to each host. |
|
189 | 196 | |
|
190 | 197 | The ``--clusterfile`` command line option lets you specify the file to use for |
|
191 | 198 | the cluster definition. Once you have your cluster file and you can |
|
192 | 199 | :command:`ssh` into the remote hosts with out an password you are ready to |
|
193 | 200 | start your cluster like so: |
|
194 | 201 | |
|
195 | 202 | .. sourcecode:: bash |
|
196 | 203 | |
|
197 | 204 | $ ipcluster ssh --clusterfile /path/to/my/clusterfile.py |
|
198 | 205 | |
|
199 | 206 | |
|
200 | 207 | Two helper shell scripts are used to start and stop :command:`ipengine` on remote hosts: |
|
201 | 208 | |
|
202 | 209 | * sshx.sh |
|
203 | 210 | * engine_killer.sh |
|
204 | 211 | |
|
205 | 212 | Defaults for both of these are contained in the source code for :command:`ipcluster`. The default scripts are written to a local file in a tmep directory and then copied to a temp directory on the remote host and executed from there. On most Unix, Linux and OS X systems this is /tmp. |
|
206 | 213 | |
|
207 | 214 | The default sshx.sh is the following: |
|
208 | 215 | |
|
209 | 216 | .. sourcecode:: bash |
|
210 | 217 | |
|
211 | 218 | #!/bin/sh |
|
212 | 219 | "$@" &> /dev/null & |
|
213 | 220 | echo $! |
|
214 | 221 | |
|
215 | 222 | If you want to use a custom sshx.sh script you need to use the ``--sshx`` |
|
216 | 223 | option and specify the file to use. Using a custom sshx.sh file could be |
|
217 | 224 | helpful when you need to setup the environment on the remote host before |
|
218 | 225 | executing :command:`ipengine`. |
|
219 | 226 | |
|
220 | 227 | For a detailed options list: |
|
221 | 228 | |
|
222 | 229 | .. sourcecode:: bash |
|
223 | 230 | |
|
224 | 231 | $ ipcluster ssh -h |
|
225 | 232 | |
|
226 | 233 | Current limitations of the SSH mode of :command:`ipcluster` are: |
|
227 | 234 | |
|
228 | 235 | * Untested on Windows. Would require a working :command:`ssh` on Windows. |
|
229 | 236 | Also, we are using shell scripts to setup and execute commands on remote |
|
230 | 237 | hosts. |
|
231 | 238 | * :command:`ipcontroller` is started on localhost, with no option to start it |
|
232 | 239 | on a remote node. |
|
233 | 240 | |
|
234 | 241 | Using the :command:`ipcontroller` and :command:`ipengine` commands |
|
235 | 242 | ================================================================== |
|
236 | 243 | |
|
237 | 244 | It is also possible to use the :command:`ipcontroller` and :command:`ipengine` commands to start your controller and engines. This approach gives you full control over all aspects of the startup process. |
|
238 | 245 | |
|
239 | 246 | Starting the controller and engine on your local machine |
|
240 | 247 | -------------------------------------------------------- |
|
241 | 248 | |
|
242 | 249 | To use :command:`ipcontroller` and :command:`ipengine` to start things on your |
|
243 | 250 | local machine, do the following. |
|
244 | 251 | |
|
245 | 252 | First start the controller:: |
|
246 | 253 | |
|
247 | 254 | $ ipcontroller |
|
248 | 255 | |
|
249 | 256 | Next, start however many instances of the engine you want using (repeatedly) the command:: |
|
250 | 257 | |
|
251 | 258 | $ ipengine |
|
252 | 259 | |
|
253 | 260 | The engines should start and automatically connect to the controller using the FURL files in :file:`~./ipython/security`. You are now ready to use the controller and engines from IPython. |
|
254 | 261 | |
|
255 | 262 | .. warning:: |
|
256 | 263 | |
|
257 | 264 | The order of the above operations is very important. You *must* |
|
258 | 265 | start the controller before the engines, since the engines connect |
|
259 | 266 | to the controller as they get started. |
|
260 | 267 | |
|
261 | 268 | .. note:: |
|
262 | 269 | |
|
263 | 270 | On some platforms (OS X), to put the controller and engine into the |
|
264 | 271 | background you may need to give these commands in the form ``(ipcontroller |
|
265 | 272 | &)`` and ``(ipengine &)`` (with the parentheses) for them to work |
|
266 | 273 | properly. |
|
267 | 274 | |
|
268 | 275 | Starting the controller and engines on different hosts |
|
269 | 276 | ------------------------------------------------------ |
|
270 | 277 | |
|
271 | 278 | When the controller and engines are running on different hosts, things are |
|
272 | 279 | slightly more complicated, but the underlying ideas are the same: |
|
273 | 280 | |
|
274 | 281 | 1. Start the controller on a host using :command:`ipcontroller`. |
|
275 | 282 | 2. Copy :file:`ipcontroller-engine.furl` from :file:`~./ipython/security` on the controller's host to the host where the engines will run. |
|
276 | 283 | 3. Use :command:`ipengine` on the engine's hosts to start the engines. |
|
277 | 284 | |
|
278 | 285 | The only thing you have to be careful of is to tell :command:`ipengine` where the :file:`ipcontroller-engine.furl` file is located. There are two ways you can do this: |
|
279 | 286 | |
|
280 | 287 | * Put :file:`ipcontroller-engine.furl` in the :file:`~./ipython/security` |
|
281 | 288 | directory on the engine's host, where it will be found automatically. |
|
282 | 289 | * Call :command:`ipengine` with the ``--furl-file=full_path_to_the_file`` |
|
283 | 290 | flag. |
|
284 | 291 | |
|
285 | 292 | The ``--furl-file`` flag works like this:: |
|
286 | 293 | |
|
287 | 294 | $ ipengine --furl-file=/path/to/my/ipcontroller-engine.furl |
|
288 | 295 | |
|
289 | 296 | .. note:: |
|
290 | 297 | |
|
291 | 298 | If the controller's and engine's hosts all have a shared file system |
|
292 | 299 | (:file:`~./ipython/security` is the same on all of them), then things |
|
293 | 300 | will just work! |
|
294 | 301 | |
|
295 | 302 | Make FURL files persistent |
|
296 | 303 | --------------------------- |
|
297 | 304 | |
|
298 | 305 | At fist glance it may seem that that managing the FURL files is a bit annoying. Going back to the house and key analogy, copying the FURL around each time you start the controller is like having to make a new key every time you want to unlock the door and enter your house. As with your house, you want to be able to create the key (or FURL file) once, and then simply use it at any point in the future. |
|
299 | 306 | |
|
300 | 307 | This is possible. The only thing you have to do is decide what ports the controller will listen on for the engines and clients. This is done as follows:: |
|
301 | 308 | |
|
302 | 309 | $ ipcontroller -r --client-port=10101 --engine-port=10102 |
|
303 | 310 | |
|
311 | These options also work with all of the various modes of | |
|
312 | :command:`ipcluster`:: | |
|
313 | ||
|
314 | $ ipcluster local -n 2 -r --client-port=10101 --engine-port=10102 | |
|
315 | ||
|
304 | 316 | Then, just copy the furl files over the first time and you are set. You can start and stop the controller and engines any many times as you want in the future, just make sure to tell the controller to use the *same* ports. |
|
305 | 317 | |
|
306 | 318 | .. note:: |
|
307 | 319 | |
|
308 | 320 | You may ask the question: what ports does the controller listen on if you |
|
309 | 321 | don't tell is to use specific ones? The default is to use high random port |
|
310 | 322 | numbers. We do this for two reasons: i) to increase security through |
|
311 | 323 | obscurity and ii) to multiple controllers on a given host to start and |
|
312 | 324 | automatically use different ports. |
|
313 | 325 | |
|
314 | 326 | Log files |
|
315 | 327 | --------- |
|
316 | 328 | |
|
317 | 329 | All of the components of IPython have log files associated with them. |
|
318 | 330 | These log files can be extremely useful in debugging problems with |
|
319 | 331 | IPython and can be found in the directory :file:`~/.ipython/log`. Sending |
|
320 | 332 | the log files to us will often help us to debug any problems. |
|
321 | 333 | |
|
322 | 334 | |
|
323 | 335 | .. [PBS] Portable Batch System. http://www.openpbs.org/ |
|
324 | 336 | .. [SSH] SSH-Agent http://en.wikipedia.org/wiki/Ssh-agent |
@@ -1,87 +1,93 b'' | |||
|
1 | 1 | # |
|
2 | 2 | # A pair of directives for inserting content that will only appear in |
|
3 | 3 | # either html or latex. |
|
4 | 4 | # |
|
5 | 5 | |
|
6 | 6 | from docutils.nodes import Body, Element |
|
7 | 7 | from docutils.writers.html4css1 import HTMLTranslator |
|
8 | from sphinx.latexwriter import LaTeXTranslator | |
|
9 | 8 | from docutils.parsers.rst import directives |
|
10 | 9 | |
|
10 | # The sphinx API has changed, so we try both the old and new import forms | |
|
11 | try: | |
|
12 | from sphinx.latexwriter import LaTeXTranslator | |
|
13 | except ImportError: | |
|
14 | from sphinx.writers.latex import LaTeXTranslator | |
|
15 | ||
|
16 | ||
|
11 | 17 | class html_only(Body, Element): |
|
12 | 18 | pass |
|
13 | 19 | |
|
14 | 20 | class latex_only(Body, Element): |
|
15 | 21 | pass |
|
16 | 22 | |
|
17 | 23 | def run(content, node_class, state, content_offset): |
|
18 | 24 | text = '\n'.join(content) |
|
19 | 25 | node = node_class(text) |
|
20 | 26 | state.nested_parse(content, content_offset, node) |
|
21 | 27 | return [node] |
|
22 | 28 | |
|
23 | 29 | try: |
|
24 | 30 | from docutils.parsers.rst import Directive |
|
25 | 31 | except ImportError: |
|
26 | 32 | from docutils.parsers.rst.directives import _directives |
|
27 | 33 | |
|
28 | 34 | def html_only_directive(name, arguments, options, content, lineno, |
|
29 | 35 | content_offset, block_text, state, state_machine): |
|
30 | 36 | return run(content, html_only, state, content_offset) |
|
31 | 37 | |
|
32 | 38 | def latex_only_directive(name, arguments, options, content, lineno, |
|
33 | 39 | content_offset, block_text, state, state_machine): |
|
34 | 40 | return run(content, latex_only, state, content_offset) |
|
35 | 41 | |
|
36 | 42 | for func in (html_only_directive, latex_only_directive): |
|
37 | 43 | func.content = 1 |
|
38 | 44 | func.options = {} |
|
39 | 45 | func.arguments = None |
|
40 | 46 | |
|
41 | 47 | _directives['htmlonly'] = html_only_directive |
|
42 | 48 | _directives['latexonly'] = latex_only_directive |
|
43 | 49 | else: |
|
44 | 50 | class OnlyDirective(Directive): |
|
45 | 51 | has_content = True |
|
46 | 52 | required_arguments = 0 |
|
47 | 53 | optional_arguments = 0 |
|
48 | 54 | final_argument_whitespace = True |
|
49 | 55 | option_spec = {} |
|
50 | 56 | |
|
51 | 57 | def run(self): |
|
52 | 58 | self.assert_has_content() |
|
53 | 59 | return run(self.content, self.node_class, |
|
54 | 60 | self.state, self.content_offset) |
|
55 | 61 | |
|
56 | 62 | class HtmlOnlyDirective(OnlyDirective): |
|
57 | 63 | node_class = html_only |
|
58 | 64 | |
|
59 | 65 | class LatexOnlyDirective(OnlyDirective): |
|
60 | 66 | node_class = latex_only |
|
61 | 67 | |
|
62 | 68 | directives.register_directive('htmlonly', HtmlOnlyDirective) |
|
63 | 69 | directives.register_directive('latexonly', LatexOnlyDirective) |
|
64 | 70 | |
|
65 | 71 | def setup(app): |
|
66 | 72 | app.add_node(html_only) |
|
67 | 73 | app.add_node(latex_only) |
|
68 | 74 | |
|
69 | 75 | # Add visit/depart methods to HTML-Translator: |
|
70 | 76 | def visit_perform(self, node): |
|
71 | 77 | pass |
|
72 | 78 | def depart_perform(self, node): |
|
73 | 79 | pass |
|
74 | 80 | def visit_ignore(self, node): |
|
75 | 81 | node.children = [] |
|
76 | 82 | def depart_ignore(self, node): |
|
77 | 83 | node.children = [] |
|
78 | 84 | |
|
79 | 85 | HTMLTranslator.visit_html_only = visit_perform |
|
80 | 86 | HTMLTranslator.depart_html_only = depart_perform |
|
81 | 87 | HTMLTranslator.visit_latex_only = visit_ignore |
|
82 | 88 | HTMLTranslator.depart_latex_only = depart_ignore |
|
83 | 89 | |
|
84 | 90 | LaTeXTranslator.visit_html_only = visit_ignore |
|
85 | 91 | LaTeXTranslator.depart_html_only = depart_ignore |
|
86 | 92 | LaTeXTranslator.visit_latex_only = visit_perform |
|
87 | 93 | LaTeXTranslator.depart_latex_only = depart_perform |
General Comments 0
You need to be logged in to leave comments.
Login now