Show More
@@ -0,0 +1,41 b'' | |||
|
1 | """Minimal script to reproduce our nasty reference counting bug. | |
|
2 | ||
|
3 | The problem is related to https://bugs.launchpad.net/ipython/+bug/269966 | |
|
4 | ||
|
5 | The original fix for that appeared to work, but John D. Hunter found a | |
|
6 | matplotlib example which, when run twice in a row, would break. The problem | |
|
7 | were references held by open figures to internals of Tkinter. | |
|
8 | ||
|
9 | This code reproduces the problem that John saw, without matplotlib. | |
|
10 | ||
|
11 | This script is meant to be called by other parts of the test suite that call it | |
|
12 | via %run as if it were executed interactively by the user. As of 2009-04-13, | |
|
13 | test_magic.py calls it. | |
|
14 | """ | |
|
15 | ||
|
16 | #----------------------------------------------------------------------------- | |
|
17 | # Module imports | |
|
18 | #----------------------------------------------------------------------------- | |
|
19 | import sys | |
|
20 | ||
|
21 | from IPython import ipapi | |
|
22 | ||
|
23 | #----------------------------------------------------------------------------- | |
|
24 | # Globals | |
|
25 | #----------------------------------------------------------------------------- | |
|
26 | ip = ipapi.get() | |
|
27 | ||
|
28 | if not '_refbug_cache' in ip.user_ns: | |
|
29 | ip.user_ns['_refbug_cache'] = [] | |
|
30 | ||
|
31 | ||
|
32 | aglobal = 'Hello' | |
|
33 | def f(): | |
|
34 | return aglobal | |
|
35 | ||
|
36 | cache = ip.user_ns['_refbug_cache'] | |
|
37 | cache.append(f) | |
|
38 | ||
|
39 | def call_f(): | |
|
40 | for func in cache: | |
|
41 | print 'lowercased:',func().lower() |
@@ -0,0 +1,17 b'' | |||
|
1 | """Tests for the FakeModule objects. | |
|
2 | """ | |
|
3 | ||
|
4 | import nose.tools as nt | |
|
5 | ||
|
6 | from IPython.FakeModule import FakeModule, init_fakemod_dict | |
|
7 | ||
|
8 | # Make a fakemod and check a few properties | |
|
9 | def test_mk_fakemod(): | |
|
10 | fm = FakeModule() | |
|
11 | yield nt.assert_true,fm | |
|
12 | yield nt.assert_true,lambda : hasattr(fm,'__file__') | |
|
13 | ||
|
14 | def test_mk_fakemod_fromdict(): | |
|
15 | """Test making a FakeModule object with initial data""" | |
|
16 | fm = FakeModule(dict(hello=True)) | |
|
17 | nt.assert_true(fm.hello) |
@@ -15,6 +15,37 b' sessions.' | |||
|
15 | 15 | |
|
16 | 16 | import types |
|
17 | 17 | |
|
18 | def init_fakemod_dict(fm,adict=None): | |
|
19 | """Initialize a FakeModule instance __dict__. | |
|
20 | ||
|
21 | Kept as a standalone function and not a method so the FakeModule API can | |
|
22 | remain basically empty. | |
|
23 | ||
|
24 | This should be considered for private IPython use, used in managing | |
|
25 | namespaces for %run. | |
|
26 | ||
|
27 | Parameters | |
|
28 | ---------- | |
|
29 | ||
|
30 | fm : FakeModule instance | |
|
31 | ||
|
32 | adict : dict, optional | |
|
33 | """ | |
|
34 | ||
|
35 | dct = {} | |
|
36 | # It seems pydoc (and perhaps others) needs any module instance to | |
|
37 | # implement a __nonzero__ method, so we add it if missing: | |
|
38 | dct.setdefault('__nonzero__',lambda : True) | |
|
39 | dct.setdefault('__file__',__file__) | |
|
40 | ||
|
41 | if adict is not None: | |
|
42 | dct.update(adict) | |
|
43 | ||
|
44 | # Hard assignment of the object's __dict__. This is nasty but deliberate. | |
|
45 | fm.__dict__.clear() | |
|
46 | fm.__dict__.update(dct) | |
|
47 | ||
|
48 | ||
|
18 | 49 | class FakeModule(types.ModuleType): |
|
19 | 50 | """Simple class with attribute access to fake a module. |
|
20 | 51 | |
@@ -29,14 +60,7 b' class FakeModule(types.ModuleType):' | |||
|
29 | 60 | |
|
30 | 61 | # tmp to force __dict__ instance creation, else self.__dict__ fails |
|
31 | 62 | self.__iptmp = None |
|
32 | ||
|
33 | # It seems pydoc (and perhaps others) needs any module instance to | |
|
34 | # implement a __nonzero__ method, so we add it if missing: | |
|
35 | self.__dict__.setdefault('__nonzero__',lambda : True) | |
|
36 | self.__dict__.setdefault('__file__',__file__) | |
|
37 | ||
|
38 | 63 | # cleanup our temp trick |
|
39 | 64 | del self.__iptmp |
|
40 | ||
|
41 | if adict is not None: | |
|
42 | self.__dict__.update(adict) | |
|
65 | # Now, initialize the actual data in the instance dict. | |
|
66 | init_fakemod_dict(self,adict) |
@@ -1584,23 +1584,17 b' Currently the magic system has the following functions:\\n"""' | |||
|
1584 | 1584 | prog_ns = self.shell.user_ns |
|
1585 | 1585 | __name__save = self.shell.user_ns['__name__'] |
|
1586 | 1586 | prog_ns['__name__'] = '__main__' |
|
1587 |
main_mod = |
|
|
1587 | main_mod = self.shell.new_main_mod(prog_ns) | |
|
1588 | 1588 | else: |
|
1589 | 1589 | # Run in a fresh, empty namespace |
|
1590 | 1590 | if opts.has_key('n'): |
|
1591 | 1591 | name = os.path.splitext(os.path.basename(filename))[0] |
|
1592 | 1592 | else: |
|
1593 | 1593 | name = '__main__' |
|
1594 | main_mod = FakeModule() | |
|
1594 | ||
|
1595 | main_mod = self.shell.new_main_mod() | |
|
1595 | 1596 | prog_ns = main_mod.__dict__ |
|
1596 | 1597 | prog_ns['__name__'] = name |
|
1597 | ||
|
1598 | # The shell MUST hold a reference to main_mod so after %run exits, | |
|
1599 | # the python deletion mechanism doesn't zero it out (leaving | |
|
1600 | # dangling references). However, we should drop old versions of | |
|
1601 | # main_mod. There is now a proper API to manage this caching in | |
|
1602 | # the main shell object, we use that. | |
|
1603 | self.shell.cache_main_mod(main_mod) | |
|
1604 | 1598 | |
|
1605 | 1599 | # Since '%run foo' emulates 'python foo.py' at the cmd line, we must |
|
1606 | 1600 | # set the __file__ global in the script's namespace |
@@ -1703,9 +1697,14 b' Currently the magic system has the following functions:\\n"""' | |||
|
1703 | 1697 | else: |
|
1704 | 1698 | # regular execution |
|
1705 | 1699 | runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore) |
|
1700 | ||
|
1706 | 1701 | if opts.has_key('i'): |
|
1707 | 1702 | self.shell.user_ns['__name__'] = __name__save |
|
1708 | 1703 | else: |
|
1704 | # The shell MUST hold a reference to prog_ns so after %run | |
|
1705 | # exits, the python deletion mechanism doesn't zero it out | |
|
1706 | # (leaving dangling references). | |
|
1707 | self.shell.cache_main_mod(prog_ns,filename) | |
|
1709 | 1708 | # update IPython interactive namespace |
|
1710 | 1709 | del prog_ns['__name__'] |
|
1711 | 1710 | self.shell.user_ns.update(prog_ns) |
@@ -1719,6 +1718,7 b' Currently the magic system has the following functions:\\n"""' | |||
|
1719 | 1718 | # added. Otherwise it will trap references to objects |
|
1720 | 1719 | # contained therein. |
|
1721 | 1720 | del sys.modules[main_mod_name] |
|
1721 | ||
|
1722 | 1722 | self.shell.reloadhist() |
|
1723 | 1723 | |
|
1724 | 1724 | return stats |
@@ -1800,7 +1800,28 b' Currently the magic system has the following functions:\\n"""' | |||
|
1800 | 1800 | import timeit |
|
1801 | 1801 | import math |
|
1802 | 1802 | |
|
1803 | units = [u"s", u"ms", u"\xb5s", u"ns"] | |
|
1803 | # XXX: Unfortunately the unicode 'micro' symbol can cause problems in | |
|
1804 | # certain terminals. Until we figure out a robust way of | |
|
1805 | # auto-detecting if the terminal can deal with it, use plain 'us' for | |
|
1806 | # microseconds. I am really NOT happy about disabling the proper | |
|
1807 | # 'micro' prefix, but crashing is worse... If anyone knows what the | |
|
1808 | # right solution for this is, I'm all ears... | |
|
1809 | # | |
|
1810 | # Note: using | |
|
1811 | # | |
|
1812 | # s = u'\xb5' | |
|
1813 | # s.encode(sys.getdefaultencoding()) | |
|
1814 | # | |
|
1815 | # is not sufficient, as I've seen terminals where that fails but | |
|
1816 | # print s | |
|
1817 | # | |
|
1818 | # succeeds | |
|
1819 | # | |
|
1820 | # See bug: https://bugs.launchpad.net/ipython/+bug/348466 | |
|
1821 | ||
|
1822 | #units = [u"s", u"ms",u'\xb5',"ns"] | |
|
1823 | units = [u"s", u"ms",u'us',"ns"] | |
|
1824 | ||
|
1804 | 1825 | scaling = [1, 1e3, 1e6, 1e9] |
|
1805 | 1826 | |
|
1806 | 1827 | opts, stmt = self.parse_options(parameter_s,'n:r:tcp:', |
@@ -1839,9 +1860,9 b' Currently the magic system has the following functions:\\n"""' | |||
|
1839 | 1860 | # determine number so that 0.2 <= total time < 2.0 |
|
1840 | 1861 | number = 1 |
|
1841 | 1862 | for i in range(1, 10): |
|
1842 | number *= 10 | |
|
1843 | 1863 | if timer.timeit(number) >= 0.2: |
|
1844 | 1864 | break |
|
1865 | number *= 10 | |
|
1845 | 1866 | |
|
1846 | 1867 | best = min(timer.repeat(repeat, number)) / number |
|
1847 | 1868 |
@@ -128,7 +128,7 b' prompt_specials_color = {' | |||
|
128 | 128 | r'\N': '${self.cache.prompt_count}', |
|
129 | 129 | # Prompt/history count, with the actual digits replaced by dots. Used |
|
130 | 130 | # mainly in continuation prompts (prompt_in2) |
|
131 | r'\D': '${"."*len(str(self.cache.prompt_count))}', | |
|
131 | r'\D': '${"."*__builtins__.len(__builtins__.str(self.cache.prompt_count))}', | |
|
132 | 132 | # Current working directory |
|
133 | 133 | r'\w': '${os.getcwd()}', |
|
134 | 134 | # Current time |
@@ -20,10 +20,10 b" name = 'ipython'" | |||
|
20 | 20 | # because bdist_rpm does not accept dashes (an RPM) convention, and |
|
21 | 21 | # bdist_deb does not accept underscores (a Debian convention). |
|
22 | 22 | |
|
23 |
development = |
|
|
24 |
version_base = '0. |
|
|
23 | development = True # change this to False to do a release | |
|
24 | version_base = '0.10' | |
|
25 | 25 | branch = 'ipython' |
|
26 |
revision = '11 |
|
|
26 | revision = '1163' | |
|
27 | 27 | |
|
28 | 28 | if development: |
|
29 | 29 | if branch == 'ipython': |
@@ -54,7 +54,7 b' from pprint import pprint, pformat' | |||
|
54 | 54 | from IPython import Debugger,OInspect,PyColorize,ultraTB |
|
55 | 55 | from IPython.ColorANSI import ColorScheme,ColorSchemeTable # too long names |
|
56 | 56 | from IPython.Extensions import pickleshare |
|
57 | from IPython.FakeModule import FakeModule | |
|
57 | from IPython.FakeModule import FakeModule, init_fakemod_dict | |
|
58 | 58 | from IPython.Itpl import Itpl,itpl,printpl,ItplNS,itplns |
|
59 | 59 | from IPython.Logger import Logger |
|
60 | 60 | from IPython.Magic import Magic |
@@ -108,6 +108,197 b' def softspace(file, newvalue):' | |||
|
108 | 108 | return oldvalue |
|
109 | 109 | |
|
110 | 110 | |
|
111 | def user_setup(ipythondir,rc_suffix,mode='install',interactive=True): | |
|
112 | """Install or upgrade the user configuration directory. | |
|
113 | ||
|
114 | Can be called when running for the first time or to upgrade the user's | |
|
115 | .ipython/ directory. | |
|
116 | ||
|
117 | Parameters | |
|
118 | ---------- | |
|
119 | ipythondir : path | |
|
120 | The directory to be used for installation/upgrade. In 'install' mode, | |
|
121 | if this path already exists, the function exits immediately. | |
|
122 | ||
|
123 | rc_suffix : str | |
|
124 | Extension for the config files. On *nix platforms it is typically the | |
|
125 | empty string, while Windows normally uses '.ini'. | |
|
126 | ||
|
127 | mode : str, optional | |
|
128 | Valid modes are 'install' and 'upgrade'. | |
|
129 | ||
|
130 | interactive : bool, optional | |
|
131 | If False, do not wait for user input on any errors. Normally after | |
|
132 | printing its status information, this function waits for the user to | |
|
133 | hit Return before proceeding. This is because the default use case is | |
|
134 | when first installing the IPython configuration, so we want the user to | |
|
135 | acknowledge the initial message, which contains some useful | |
|
136 | information. | |
|
137 | """ | |
|
138 | ||
|
139 | # For automatic use, deactivate all i/o | |
|
140 | if interactive: | |
|
141 | def wait(): | |
|
142 | try: | |
|
143 | raw_input("Please press <RETURN> to start IPython.") | |
|
144 | except EOFError: | |
|
145 | print >> Term.cout | |
|
146 | print '*'*70 | |
|
147 | ||
|
148 | def printf(s): | |
|
149 | print s | |
|
150 | else: | |
|
151 | wait = lambda : None | |
|
152 | printf = lambda s : None | |
|
153 | ||
|
154 | # Install mode should be re-entrant: if the install dir already exists, | |
|
155 | # bail out cleanly | |
|
156 | if mode == 'install' and os.path.isdir(ipythondir): | |
|
157 | return | |
|
158 | ||
|
159 | cwd = os.getcwd() # remember where we started | |
|
160 | glb = glob.glob | |
|
161 | ||
|
162 | printf('*'*70) | |
|
163 | if mode == 'install': | |
|
164 | printf( | |
|
165 | """Welcome to IPython. I will try to create a personal configuration directory | |
|
166 | where you can customize many aspects of IPython's functionality in:\n""") | |
|
167 | else: | |
|
168 | printf('I am going to upgrade your configuration in:') | |
|
169 | ||
|
170 | printf(ipythondir) | |
|
171 | ||
|
172 | rcdirend = os.path.join('IPython','UserConfig') | |
|
173 | cfg = lambda d: os.path.join(d,rcdirend) | |
|
174 | try: | |
|
175 | rcdir = filter(os.path.isdir,map(cfg,sys.path))[0] | |
|
176 | printf("Initializing from configuration: %s" % rcdir) | |
|
177 | except IndexError: | |
|
178 | warning = """ | |
|
179 | Installation error. IPython's directory was not found. | |
|
180 | ||
|
181 | Check the following: | |
|
182 | ||
|
183 | The ipython/IPython directory should be in a directory belonging to your | |
|
184 | PYTHONPATH environment variable (that is, it should be in a directory | |
|
185 | belonging to sys.path). You can copy it explicitly there or just link to it. | |
|
186 | ||
|
187 | IPython will create a minimal default configuration for you. | |
|
188 | ||
|
189 | """ | |
|
190 | warn(warning) | |
|
191 | wait() | |
|
192 | ||
|
193 | if sys.platform =='win32': | |
|
194 | inif = 'ipythonrc.ini' | |
|
195 | else: | |
|
196 | inif = 'ipythonrc' | |
|
197 | minimal_setup = {'ipy_user_conf.py' : 'import ipy_defaults', | |
|
198 | inif : '# intentionally left blank' } | |
|
199 | os.makedirs(ipythondir, mode = 0777) | |
|
200 | for f, cont in minimal_setup.items(): | |
|
201 | # In 2.5, this can be more cleanly done using 'with' | |
|
202 | fobj = file(ipythondir + '/' + f,'w') | |
|
203 | fobj.write(cont) | |
|
204 | fobj.close() | |
|
205 | ||
|
206 | return | |
|
207 | ||
|
208 | if mode == 'install': | |
|
209 | try: | |
|
210 | shutil.copytree(rcdir,ipythondir) | |
|
211 | os.chdir(ipythondir) | |
|
212 | rc_files = glb("ipythonrc*") | |
|
213 | for rc_file in rc_files: | |
|
214 | os.rename(rc_file,rc_file+rc_suffix) | |
|
215 | except: | |
|
216 | warning = """ | |
|
217 | ||
|
218 | There was a problem with the installation: | |
|
219 | %s | |
|
220 | Try to correct it or contact the developers if you think it's a bug. | |
|
221 | IPython will proceed with builtin defaults.""" % sys.exc_info()[1] | |
|
222 | warn(warning) | |
|
223 | wait() | |
|
224 | return | |
|
225 | ||
|
226 | elif mode == 'upgrade': | |
|
227 | try: | |
|
228 | os.chdir(ipythondir) | |
|
229 | except: | |
|
230 | printf(""" | |
|
231 | Can not upgrade: changing to directory %s failed. Details: | |
|
232 | %s | |
|
233 | """ % (ipythondir,sys.exc_info()[1]) ) | |
|
234 | wait() | |
|
235 | return | |
|
236 | else: | |
|
237 | sources = glb(os.path.join(rcdir,'[A-Za-z]*')) | |
|
238 | for new_full_path in sources: | |
|
239 | new_filename = os.path.basename(new_full_path) | |
|
240 | if new_filename.startswith('ipythonrc'): | |
|
241 | new_filename = new_filename + rc_suffix | |
|
242 | # The config directory should only contain files, skip any | |
|
243 | # directories which may be there (like CVS) | |
|
244 | if os.path.isdir(new_full_path): | |
|
245 | continue | |
|
246 | if os.path.exists(new_filename): | |
|
247 | old_file = new_filename+'.old' | |
|
248 | if os.path.exists(old_file): | |
|
249 | os.remove(old_file) | |
|
250 | os.rename(new_filename,old_file) | |
|
251 | shutil.copy(new_full_path,new_filename) | |
|
252 | else: | |
|
253 | raise ValueError('unrecognized mode for install: %r' % mode) | |
|
254 | ||
|
255 | # Fix line-endings to those native to each platform in the config | |
|
256 | # directory. | |
|
257 | try: | |
|
258 | os.chdir(ipythondir) | |
|
259 | except: | |
|
260 | printf(""" | |
|
261 | Problem: changing to directory %s failed. | |
|
262 | Details: | |
|
263 | %s | |
|
264 | ||
|
265 | Some configuration files may have incorrect line endings. This should not | |
|
266 | cause any problems during execution. """ % (ipythondir,sys.exc_info()[1]) ) | |
|
267 | wait() | |
|
268 | else: | |
|
269 | for fname in glb('ipythonrc*'): | |
|
270 | try: | |
|
271 | native_line_ends(fname,backup=0) | |
|
272 | except IOError: | |
|
273 | pass | |
|
274 | ||
|
275 | if mode == 'install': | |
|
276 | printf(""" | |
|
277 | Successful installation! | |
|
278 | ||
|
279 | Please read the sections 'Initial Configuration' and 'Quick Tips' in the | |
|
280 | IPython manual (there are both HTML and PDF versions supplied with the | |
|
281 | distribution) to make sure that your system environment is properly configured | |
|
282 | to take advantage of IPython's features. | |
|
283 | ||
|
284 | Important note: the configuration system has changed! The old system is | |
|
285 | still in place, but its setting may be partly overridden by the settings in | |
|
286 | "~/.ipython/ipy_user_conf.py" config file. Please take a look at the file | |
|
287 | if some of the new settings bother you. | |
|
288 | ||
|
289 | """) | |
|
290 | else: | |
|
291 | printf(""" | |
|
292 | Successful upgrade! | |
|
293 | ||
|
294 | All files in your directory: | |
|
295 | %(ipythondir)s | |
|
296 | which would have been overwritten by the upgrade were backed up with a .old | |
|
297 | extension. If you had made particular customizations in those files you may | |
|
298 | want to merge them back into the new files.""" % locals() ) | |
|
299 | wait() | |
|
300 | os.chdir(cwd) | |
|
301 | ||
|
111 | 302 | #**************************************************************************** |
|
112 | 303 | # Local use exceptions |
|
113 | 304 | class SpaceInInput(exceptions.Exception): pass |
@@ -308,13 +499,24 b' class InteractiveShell(object,Magic):' | |||
|
308 | 499 | # calling functions defined in the script that use other things from |
|
309 | 500 | # the script will fail, because the function's closure had references |
|
310 | 501 | # to the original objects, which are now all None. So we must protect |
|
311 |
# these modules from deletion by keeping a cache. |
|
|
312 | # stale modules around (we only need the one from the last run), we use | |
|
313 | # a dict keyed with the full path to the script, so only the last | |
|
314 | # version of the module is held in the cache. The %reset command will | |
|
315 | # flush this cache. See the cache_main_mod() and clear_main_mod_cache() | |
|
316 | # methods for details on use. | |
|
317 | self._user_main_modules = {} | |
|
502 | # these modules from deletion by keeping a cache. | |
|
503 | # | |
|
504 | # To avoid keeping stale modules around (we only need the one from the | |
|
505 | # last run), we use a dict keyed with the full path to the script, so | |
|
506 | # only the last version of the module is held in the cache. Note, | |
|
507 | # however, that we must cache the module *namespace contents* (their | |
|
508 | # __dict__). Because if we try to cache the actual modules, old ones | |
|
509 | # (uncached) could be destroyed while still holding references (such as | |
|
510 | # those held by GUI objects that tend to be long-lived)> | |
|
511 | # | |
|
512 | # The %reset command will flush this cache. See the cache_main_mod() | |
|
513 | # and clear_main_mod_cache() methods for details on use. | |
|
514 | ||
|
515 | # This is the cache used for 'main' namespaces | |
|
516 | self._main_ns_cache = {} | |
|
517 | # And this is the single instance of FakeModule whose __dict__ we keep | |
|
518 | # copying and clearing for reuse on each %run | |
|
519 | self._user_main_module = FakeModule() | |
|
318 | 520 | |
|
319 | 521 | # A table holding all the namespaces IPython deals with, so that |
|
320 | 522 | # introspection facilities can search easily. |
@@ -330,7 +532,7 b' class InteractiveShell(object,Magic):' | |||
|
330 | 532 | # a simple list. |
|
331 | 533 | self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns, |
|
332 | 534 | self.alias_table, self.internal_ns, |
|
333 |
self. |
|
|
535 | self._main_ns_cache ] | |
|
334 | 536 | |
|
335 | 537 | # We need to insert into sys.modules something that looks like a |
|
336 | 538 | # module but which accesses the IPython namespace, for shelve and |
@@ -1114,156 +1316,11 b' class InteractiveShell(object,Magic):' | |||
|
1114 | 1316 | def user_setup(self,ipythondir,rc_suffix,mode='install'): |
|
1115 | 1317 | """Install the user configuration directory. |
|
1116 | 1318 | |
|
1117 | Can be called when running for the first time or to upgrade the user's | |
|
1118 | .ipython/ directory with the mode parameter. Valid modes are 'install' | |
|
1119 | and 'upgrade'.""" | |
|
1120 | ||
|
1121 | def wait(): | |
|
1122 | try: | |
|
1123 | raw_input("Please press <RETURN> to start IPython.") | |
|
1124 | except EOFError: | |
|
1125 | print >> Term.cout | |
|
1126 | print '*'*70 | |
|
1127 | ||
|
1128 | cwd = os.getcwd() # remember where we started | |
|
1129 | glb = glob.glob | |
|
1130 | print '*'*70 | |
|
1131 | if mode == 'install': | |
|
1132 | print \ | |
|
1133 | """Welcome to IPython. I will try to create a personal configuration directory | |
|
1134 | where you can customize many aspects of IPython's functionality in:\n""" | |
|
1135 | else: | |
|
1136 | print 'I am going to upgrade your configuration in:' | |
|
1137 | ||
|
1138 | print ipythondir | |
|
1139 | ||
|
1140 | rcdirend = os.path.join('IPython','UserConfig') | |
|
1141 | cfg = lambda d: os.path.join(d,rcdirend) | |
|
1142 | try: | |
|
1143 | rcdir = filter(os.path.isdir,map(cfg,sys.path))[0] | |
|
1144 | print "Initializing from configuration",rcdir | |
|
1145 | except IndexError: | |
|
1146 | warning = """ | |
|
1147 | Installation error. IPython's directory was not found. | |
|
1148 | ||
|
1149 | Check the following: | |
|
1150 | ||
|
1151 | The ipython/IPython directory should be in a directory belonging to your | |
|
1152 | PYTHONPATH environment variable (that is, it should be in a directory | |
|
1153 | belonging to sys.path). You can copy it explicitly there or just link to it. | |
|
1154 | ||
|
1155 | IPython will create a minimal default configuration for you. | |
|
1156 | ||
|
1157 | """ | |
|
1158 | warn(warning) | |
|
1159 | wait() | |
|
1160 | ||
|
1161 | if sys.platform =='win32': | |
|
1162 | inif = 'ipythonrc.ini' | |
|
1163 | else: | |
|
1164 | inif = 'ipythonrc' | |
|
1165 | minimal_setup = {'ipy_user_conf.py' : 'import ipy_defaults', | |
|
1166 | inif : '# intentionally left blank' } | |
|
1167 | os.makedirs(ipythondir, mode = 0777) | |
|
1168 | for f, cont in minimal_setup.items(): | |
|
1169 | open(ipythondir + '/' + f,'w').write(cont) | |
|
1170 | ||
|
1171 | return | |
|
1172 | ||
|
1173 | if mode == 'install': | |
|
1174 | try: | |
|
1175 | shutil.copytree(rcdir,ipythondir) | |
|
1176 | os.chdir(ipythondir) | |
|
1177 | rc_files = glb("ipythonrc*") | |
|
1178 | for rc_file in rc_files: | |
|
1179 | os.rename(rc_file,rc_file+rc_suffix) | |
|
1180 | except: | |
|
1181 | warning = """ | |
|
1182 | ||
|
1183 | There was a problem with the installation: | |
|
1184 | %s | |
|
1185 | Try to correct it or contact the developers if you think it's a bug. | |
|
1186 | IPython will proceed with builtin defaults.""" % sys.exc_info()[1] | |
|
1187 | warn(warning) | |
|
1188 | wait() | |
|
1189 | return | |
|
1190 | ||
|
1191 | elif mode == 'upgrade': | |
|
1192 | try: | |
|
1193 | os.chdir(ipythondir) | |
|
1194 | except: | |
|
1195 | print """ | |
|
1196 | Can not upgrade: changing to directory %s failed. Details: | |
|
1197 | %s | |
|
1198 | """ % (ipythondir,sys.exc_info()[1]) | |
|
1199 | wait() | |
|
1200 | return | |
|
1201 | else: | |
|
1202 | sources = glb(os.path.join(rcdir,'[A-Za-z]*')) | |
|
1203 | for new_full_path in sources: | |
|
1204 | new_filename = os.path.basename(new_full_path) | |
|
1205 | if new_filename.startswith('ipythonrc'): | |
|
1206 | new_filename = new_filename + rc_suffix | |
|
1207 | # The config directory should only contain files, skip any | |
|
1208 | # directories which may be there (like CVS) | |
|
1209 | if os.path.isdir(new_full_path): | |
|
1210 | continue | |
|
1211 | if os.path.exists(new_filename): | |
|
1212 | old_file = new_filename+'.old' | |
|
1213 | if os.path.exists(old_file): | |
|
1214 | os.remove(old_file) | |
|
1215 | os.rename(new_filename,old_file) | |
|
1216 | shutil.copy(new_full_path,new_filename) | |
|
1217 | else: | |
|
1218 | raise ValueError,'unrecognized mode for install:',`mode` | |
|
1219 | ||
|
1220 | # Fix line-endings to those native to each platform in the config | |
|
1221 | # directory. | |
|
1222 | try: | |
|
1223 | os.chdir(ipythondir) | |
|
1224 | except: | |
|
1225 | print """ | |
|
1226 | Problem: changing to directory %s failed. | |
|
1227 | Details: | |
|
1228 | %s | |
|
1229 | ||
|
1230 | Some configuration files may have incorrect line endings. This should not | |
|
1231 | cause any problems during execution. """ % (ipythondir,sys.exc_info()[1]) | |
|
1232 | wait() | |
|
1233 | else: | |
|
1234 | for fname in glb('ipythonrc*'): | |
|
1235 | try: | |
|
1236 | native_line_ends(fname,backup=0) | |
|
1237 | except IOError: | |
|
1238 | pass | |
|
1239 | ||
|
1240 | if mode == 'install': | |
|
1241 | print """ | |
|
1242 | Successful installation! | |
|
1243 | ||
|
1244 | Please read the sections 'Initial Configuration' and 'Quick Tips' in the | |
|
1245 | IPython manual (there are both HTML and PDF versions supplied with the | |
|
1246 | distribution) to make sure that your system environment is properly configured | |
|
1247 | to take advantage of IPython's features. | |
|
1248 | ||
|
1249 | Important note: the configuration system has changed! The old system is | |
|
1250 | still in place, but its setting may be partly overridden by the settings in | |
|
1251 | "~/.ipython/ipy_user_conf.py" config file. Please take a look at the file | |
|
1252 | if some of the new settings bother you. | |
|
1253 | ||
|
1254 | """ | |
|
1255 | else: | |
|
1256 | print """ | |
|
1257 | Successful upgrade! | |
|
1258 | ||
|
1259 | All files in your directory: | |
|
1260 | %(ipythondir)s | |
|
1261 | which would have been overwritten by the upgrade were backed up with a .old | |
|
1262 | extension. If you had made particular customizations in those files you may | |
|
1263 | want to merge them back into the new files.""" % locals() | |
|
1264 | wait() | |
|
1265 | os.chdir(cwd) | |
|
1266 | # end user_setup() | |
|
1319 | Note | |
|
1320 | ---- | |
|
1321 | DEPRECATED: use the top-level user_setup() function instead. | |
|
1322 | """ | |
|
1323 | return user_setup(ipythondir,rc_suffix,mode) | |
|
1267 | 1324 | |
|
1268 | 1325 | def atexit_operations(self): |
|
1269 | 1326 | """This will be executed at the time of exit. |
@@ -1441,35 +1498,53 b' want to merge them back into the new files.""" % locals()' | |||
|
1441 | 1498 | return True |
|
1442 | 1499 | return ask_yes_no(prompt,default) |
|
1443 | 1500 | |
|
1444 |
def |
|
|
1445 | """Cache a main module. | |
|
1501 | def new_main_mod(self,ns=None): | |
|
1502 | """Return a new 'main' module object for user code execution. | |
|
1503 | """ | |
|
1504 | main_mod = self._user_main_module | |
|
1505 | init_fakemod_dict(main_mod,ns) | |
|
1506 | return main_mod | |
|
1507 | ||
|
1508 | def cache_main_mod(self,ns,fname): | |
|
1509 | """Cache a main module's namespace. | |
|
1446 | 1510 | |
|
1447 |
When scripts are executed via %run, we must keep a reference to the |
|
|
1448 |
__main__ module (a FakeModule instance) around so |
|
|
1449 |
clear it, rendering objects defined therein |
|
|
1511 | When scripts are executed via %run, we must keep a reference to the | |
|
1512 | namespace of their __main__ module (a FakeModule instance) around so | |
|
1513 | that Python doesn't clear it, rendering objects defined therein | |
|
1514 | useless. | |
|
1450 | 1515 | |
|
1451 | 1516 | This method keeps said reference in a private dict, keyed by the |
|
1452 | 1517 | absolute path of the module object (which corresponds to the script |
|
1453 | 1518 | path). This way, for multiple executions of the same script we only |
|
1454 |
keep one copy of |
|
|
1455 |
from old references while allowing the objects from the last |
|
|
1456 | to be accessible. | |
|
1519 | keep one copy of the namespace (the last one), thus preventing memory | |
|
1520 | leaks from old references while allowing the objects from the last | |
|
1521 | execution to be accessible. | |
|
1522 | ||
|
1523 | Note: we can not allow the actual FakeModule instances to be deleted, | |
|
1524 | because of how Python tears down modules (it hard-sets all their | |
|
1525 | references to None without regard for reference counts). This method | |
|
1526 | must therefore make a *copy* of the given namespace, to allow the | |
|
1527 | original module's __dict__ to be cleared and reused. | |
|
1457 | 1528 | |
|
1529 | ||
|
1458 | 1530 | Parameters |
|
1459 | 1531 | ---------- |
|
1460 | mod : a module object | |
|
1532 | ns : a namespace (a dict, typically) | |
|
1533 | ||
|
1534 | fname : str | |
|
1535 | Filename associated with the namespace. | |
|
1461 | 1536 | |
|
1462 | 1537 | Examples |
|
1463 | 1538 | -------- |
|
1464 | 1539 | |
|
1465 | 1540 | In [10]: import IPython |
|
1466 | 1541 | |
|
1467 | In [11]: _ip.IP.cache_main_mod(IPython) | |
|
1542 | In [11]: _ip.IP.cache_main_mod(IPython.__dict__,IPython.__file__) | |
|
1468 | 1543 | |
|
1469 |
In [12]: IPython.__file__ in _ip.IP._ |
|
|
1544 | In [12]: IPython.__file__ in _ip.IP._main_ns_cache | |
|
1470 | 1545 | Out[12]: True |
|
1471 | 1546 | """ |
|
1472 |
self. |
|
|
1547 | self._main_ns_cache[os.path.abspath(fname)] = ns.copy() | |
|
1473 | 1548 | |
|
1474 | 1549 | def clear_main_mod_cache(self): |
|
1475 | 1550 | """Clear the cache of main modules. |
@@ -1481,17 +1556,17 b' want to merge them back into the new files.""" % locals()' | |||
|
1481 | 1556 | |
|
1482 | 1557 | In [15]: import IPython |
|
1483 | 1558 | |
|
1484 | In [16]: _ip.IP.cache_main_mod(IPython) | |
|
1559 | In [16]: _ip.IP.cache_main_mod(IPython.__dict__,IPython.__file__) | |
|
1485 | 1560 | |
|
1486 |
In [17]: len(_ip.IP._ |
|
|
1561 | In [17]: len(_ip.IP._main_ns_cache) > 0 | |
|
1487 | 1562 | Out[17]: True |
|
1488 | 1563 | |
|
1489 | 1564 | In [18]: _ip.IP.clear_main_mod_cache() |
|
1490 | 1565 | |
|
1491 |
In [19]: len(_ip.IP._ |
|
|
1566 | In [19]: len(_ip.IP._main_ns_cache) == 0 | |
|
1492 | 1567 | Out[19]: True |
|
1493 | 1568 | """ |
|
1494 |
self. |
|
|
1569 | self._main_ns_cache.clear() | |
|
1495 | 1570 | |
|
1496 | 1571 | def _should_recompile(self,e): |
|
1497 | 1572 | """Utility routine for edit_syntax_error""" |
@@ -885,7 +885,62 b' class FullBlockingMultiEngineClient(InteractiveMultiEngineClient):' | |||
|
885 | 885 | targets, block = self._findTargetsAndBlock(targets, block) |
|
886 | 886 | return self._blockFromThread(self.smultiengine.run, filename, |
|
887 | 887 | targets=targets, block=block) |
|
888 | ||
|
889 | def benchmark(self, push_size=10000): | |
|
890 | """ | |
|
891 | Run performance benchmarks for the current IPython cluster. | |
|
892 | ||
|
893 | This method tests both the latency of sending command and data to the | |
|
894 | engines as well as the throughput of sending large objects to the | |
|
895 | engines using push. The latency is measured by having one or more | |
|
896 | engines execute the command 'pass'. The throughput is measure by | |
|
897 | sending an NumPy array of size `push_size` to one or more engines. | |
|
898 | ||
|
899 | These benchmarks will vary widely on different hardware and networks | |
|
900 | and thus can be used to get an idea of the performance characteristics | |
|
901 | of a particular configuration of an IPython controller and engines. | |
|
902 | ||
|
903 | This function is not testable within our current testing framework. | |
|
904 | """ | |
|
905 | import timeit, __builtin__ | |
|
906 | __builtin__._mec_self = self | |
|
907 | benchmarks = {} | |
|
908 | repeat = 3 | |
|
909 | count = 10 | |
|
910 | ||
|
911 | timer = timeit.Timer('_mec_self.execute("pass",0)') | |
|
912 | result = 1000*min(timer.repeat(repeat,count))/count | |
|
913 | benchmarks['single_engine_latency'] = (result,'msec') | |
|
914 | ||
|
915 | timer = timeit.Timer('_mec_self.execute("pass")') | |
|
916 | result = 1000*min(timer.repeat(repeat,count))/count | |
|
917 | benchmarks['all_engine_latency'] = (result,'msec') | |
|
888 | 918 | |
|
919 | try: | |
|
920 | import numpy as np | |
|
921 | except: | |
|
922 | pass | |
|
923 | else: | |
|
924 | timer = timeit.Timer( | |
|
925 | "_mec_self.push(d)", | |
|
926 | "import numpy as np; d = dict(a=np.zeros(%r,dtype='float64'))" % push_size | |
|
927 | ) | |
|
928 | result = min(timer.repeat(repeat,count))/count | |
|
929 | benchmarks['all_engine_push'] = (1e-6*push_size*8/result, 'MB/sec') | |
|
930 | ||
|
931 | try: | |
|
932 | import numpy as np | |
|
933 | except: | |
|
934 | pass | |
|
935 | else: | |
|
936 | timer = timeit.Timer( | |
|
937 | "_mec_self.push(d,0)", | |
|
938 | "import numpy as np; d = dict(a=np.zeros(%r,dtype='float64'))" % push_size | |
|
939 | ) | |
|
940 | result = min(timer.repeat(repeat,count))/count | |
|
941 | benchmarks['single_engine_push'] = (1e-6*push_size*8/result, 'MB/sec') | |
|
942 | ||
|
943 | return benchmarks | |
|
889 | 944 | |
|
890 | 945 | |
|
891 | 946 | components.registerAdapter(FullBlockingMultiEngineClient, |
@@ -478,15 +478,31 b' Try running ipcluster with the -xy flags: ipcluster local -xy -n 4""")' | |||
|
478 | 478 | cont_args.append('-y') |
|
479 | 479 | return True |
|
480 | 480 | |
|
481 | def check_reuse(args, cont_args): | |
|
482 | if args.r: | |
|
483 | cont_args.append('-r') | |
|
484 | if args.client_port == 0 or args.engine_port == 0: | |
|
485 | log.err(""" | |
|
486 | To reuse FURL files, you must also set the client and engine ports using | |
|
487 | the --client-port and --engine-port options.""") | |
|
488 | reactor.stop() | |
|
489 | return False | |
|
490 | cont_args.append('--client-port=%i' % args.client_port) | |
|
491 | cont_args.append('--engine-port=%i' % args.engine_port) | |
|
492 | return True | |
|
481 | 493 | |
|
482 | 494 | def main_local(args): |
|
483 | 495 | cont_args = [] |
|
484 | 496 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) |
|
485 | ||
|
497 | ||
|
486 | 498 | # Check security settings before proceeding |
|
487 | 499 | if not check_security(args, cont_args): |
|
488 | 500 | return |
|
489 | ||
|
501 | ||
|
502 | # See if we are reusing FURL files | |
|
503 | if not check_reuse(args, cont_args): | |
|
504 | return | |
|
505 | ||
|
490 | 506 | cl = ControllerLauncher(extra_args=cont_args) |
|
491 | 507 | dstart = cl.start() |
|
492 | 508 | def start_engines(cont_pid): |
@@ -513,18 +529,22 b' def main_local(args):' | |||
|
513 | 529 | dstart.addErrback(lambda f: f.raiseException()) |
|
514 | 530 | |
|
515 | 531 | |
|
516 |
def main_mpi |
|
|
532 | def main_mpi(args): | |
|
517 | 533 | cont_args = [] |
|
518 | 534 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) |
|
519 | ||
|
535 | ||
|
520 | 536 | # Check security settings before proceeding |
|
521 | 537 | if not check_security(args, cont_args): |
|
522 | 538 | return |
|
523 | ||
|
539 | ||
|
540 | # See if we are reusing FURL files | |
|
541 | if not check_reuse(args, cont_args): | |
|
542 | return | |
|
543 | ||
|
524 | 544 | cl = ControllerLauncher(extra_args=cont_args) |
|
525 | 545 | dstart = cl.start() |
|
526 | 546 | def start_engines(cont_pid): |
|
527 |
raw_args = [ |
|
|
547 | raw_args = [args.cmd] | |
|
528 | 548 | raw_args.extend(['-n',str(args.n)]) |
|
529 | 549 | raw_args.append('ipengine') |
|
530 | 550 | raw_args.append('-l') |
@@ -554,11 +574,15 b' def main_mpirun(args):' | |||
|
554 | 574 | def main_pbs(args): |
|
555 | 575 | cont_args = [] |
|
556 | 576 | cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller')) |
|
557 | ||
|
577 | ||
|
558 | 578 | # Check security settings before proceeding |
|
559 | 579 | if not check_security(args, cont_args): |
|
560 | 580 | return |
|
561 | ||
|
581 | ||
|
582 | # See if we are reusing FURL files | |
|
583 | if not check_reuse(args, cont_args): | |
|
584 | return | |
|
585 | ||
|
562 | 586 | cl = ControllerLauncher(extra_args=cont_args) |
|
563 | 587 | dstart = cl.start() |
|
564 | 588 | def start_engines(r): |
@@ -598,13 +622,16 b' def main_ssh(args):' | |||
|
598 | 622 | if not check_security(args, cont_args): |
|
599 | 623 | return |
|
600 | 624 | |
|
625 | # See if we are reusing FURL files | |
|
626 | if not check_reuse(args, cont_args): | |
|
627 | return | |
|
628 | ||
|
601 | 629 | cl = ControllerLauncher(extra_args=cont_args) |
|
602 | 630 | dstart = cl.start() |
|
603 | 631 | def start_engines(cont_pid): |
|
604 | 632 | ssh_set = SSHEngineSet(clusterfile['engines'], sshx=args.sshx) |
|
605 | 633 | def shutdown(signum, frame): |
|
606 | 634 | d = ssh_set.kill() |
|
607 | # d.addErrback(log.err) | |
|
608 | 635 | cl.interrupt_then_kill(1.0) |
|
609 | 636 | reactor.callLater(2.0, reactor.stop) |
|
610 | 637 | signal.signal(signal.SIGINT,shutdown) |
@@ -621,6 +648,26 b' def main_ssh(args):' | |||
|
621 | 648 | def get_args(): |
|
622 | 649 | base_parser = argparse.ArgumentParser(add_help=False) |
|
623 | 650 | base_parser.add_argument( |
|
651 | '-r', | |
|
652 | action='store_true', | |
|
653 | dest='r', | |
|
654 | help='try to reuse FURL files. Use with --client-port and --engine-port' | |
|
655 | ) | |
|
656 | base_parser.add_argument( | |
|
657 | '--client-port', | |
|
658 | type=int, | |
|
659 | dest='client_port', | |
|
660 | help='the port the controller will listen on for client connections', | |
|
661 | default=0 | |
|
662 | ) | |
|
663 | base_parser.add_argument( | |
|
664 | '--engine-port', | |
|
665 | type=int, | |
|
666 | dest='engine_port', | |
|
667 | help='the port the controller will listen on for engine connections', | |
|
668 | default=0 | |
|
669 | ) | |
|
670 | base_parser.add_argument( | |
|
624 | 671 | '-x', |
|
625 | 672 | action='store_true', |
|
626 | 673 | dest='x', |
@@ -665,7 +712,7 b' def get_args():' | |||
|
665 | 712 | |
|
666 | 713 | parser_mpirun = subparsers.add_parser( |
|
667 | 714 | 'mpirun', |
|
668 | help='run a cluster using mpirun', | |
|
715 | help='run a cluster using mpirun (mpiexec also works)', | |
|
669 | 716 | parents=[base_parser] |
|
670 | 717 | ) |
|
671 | 718 | parser_mpirun.add_argument( |
@@ -674,7 +721,20 b' def get_args():' | |||
|
674 | 721 | dest="mpi", # Don't put a default here to allow no MPI support |
|
675 | 722 | help="how to call MPI_Init (default=mpi4py)" |
|
676 | 723 | ) |
|
677 | parser_mpirun.set_defaults(func=main_mpirun) | |
|
724 | parser_mpirun.set_defaults(func=main_mpi, cmd='mpirun') | |
|
725 | ||
|
726 | parser_mpiexec = subparsers.add_parser( | |
|
727 | 'mpiexec', | |
|
728 | help='run a cluster using mpiexec (mpirun also works)', | |
|
729 | parents=[base_parser] | |
|
730 | ) | |
|
731 | parser_mpiexec.add_argument( | |
|
732 | "--mpi", | |
|
733 | type=str, | |
|
734 | dest="mpi", # Don't put a default here to allow no MPI support | |
|
735 | help="how to call MPI_Init (default=mpi4py)" | |
|
736 | ) | |
|
737 | parser_mpiexec.set_defaults(func=main_mpi, cmd='mpiexec') | |
|
678 | 738 | |
|
679 | 739 | parser_pbs = subparsers.add_parser( |
|
680 | 740 | 'pbs', |
@@ -61,7 +61,7 b' def main():' | |||
|
61 | 61 | # plugin needs to be gone through with a fine |
|
62 | 62 | # toothed comb to find what is causing the problem. |
|
63 | 63 | # '--with-ipdoctest', |
|
64 | '--doctest-tests','--doctest-extension=txt', | |
|
64 | '--ipdoctest-tests','--ipdoctest-extension=txt', | |
|
65 | 65 | '--detailed-errors', |
|
66 | 66 | |
|
67 | 67 | # We add --exe because of setuptools' imbecility (it |
@@ -81,11 +81,13 b' def main():' | |||
|
81 | 81 | (':' in arg and '.py' in arg): |
|
82 | 82 | has_tests = True |
|
83 | 83 | break |
|
84 | ||
|
84 | 85 | # If nothing was specifically requested, test full IPython |
|
85 | 86 | if not has_tests: |
|
86 | 87 | argv.append('IPython') |
|
87 | 88 | |
|
88 |
# Construct list of plugins, omitting the existing doctest plugin |
|
|
89 | # Construct list of plugins, omitting the existing doctest plugin, which | |
|
90 | # ours replaces (and extends). | |
|
89 | 91 | plugins = [IPythonDoctest(EXCLUDE)] |
|
90 | 92 | for p in nose.plugins.builtin.plugins: |
|
91 | 93 | plug = p() |
@@ -15,7 +15,6 b' Limitations:' | |||
|
15 | 15 | won't even have these special _NN variables set at all. |
|
16 | 16 | """ |
|
17 | 17 | |
|
18 | ||
|
19 | 18 | #----------------------------------------------------------------------------- |
|
20 | 19 | # Module imports |
|
21 | 20 | |
@@ -123,6 +122,13 b' class ipnsdict(dict):' | |||
|
123 | 122 | def start_ipython(): |
|
124 | 123 | """Start a global IPython shell, which we need for IPython-specific syntax. |
|
125 | 124 | """ |
|
125 | ||
|
126 | # This function should only ever run once! | |
|
127 | if hasattr(start_ipython,'already_called'): | |
|
128 | return | |
|
129 | start_ipython.already_called = True | |
|
130 | ||
|
131 | # Ok, first time we're called, go ahead | |
|
126 | 132 | import new |
|
127 | 133 | |
|
128 | 134 | import IPython |
@@ -691,6 +697,7 b' class ExtensionDoctest(doctests.Doctest):' | |||
|
691 | 697 | to exclude any filename which matches them from inclusion in the test |
|
692 | 698 | suite (using pattern.search(), NOT pattern.match() ). |
|
693 | 699 | """ |
|
700 | ||
|
694 | 701 | if exclude_patterns is None: |
|
695 | 702 | exclude_patterns = [] |
|
696 | 703 | self.exclude_patterns = map(re.compile,exclude_patterns) |
@@ -836,15 +843,33 b' class IPythonDoctest(ExtensionDoctest):' | |||
|
836 | 843 | optionflags=optionflags, |
|
837 | 844 | checker=self.checker) |
|
838 | 845 | |
|
839 | def configure(self, options, config): | |
|
846 | def options(self, parser, env=os.environ): | |
|
847 | Plugin.options(self, parser, env) | |
|
848 | parser.add_option('--ipdoctest-tests', action='store_true', | |
|
849 | dest='ipdoctest_tests', | |
|
850 | default=env.get('NOSE_IPDOCTEST_TESTS',True), | |
|
851 | help="Also look for doctests in test modules. " | |
|
852 | "Note that classes, methods and functions should " | |
|
853 | "have either doctests or non-doctest tests, " | |
|
854 | "not both. [NOSE_IPDOCTEST_TESTS]") | |
|
855 | parser.add_option('--ipdoctest-extension', action="append", | |
|
856 | dest="ipdoctest_extension", | |
|
857 | help="Also look for doctests in files with " | |
|
858 | "this extension [NOSE_IPDOCTEST_EXTENSION]") | |
|
859 | # Set the default as a list, if given in env; otherwise | |
|
860 | # an additional value set on the command line will cause | |
|
861 | # an error. | |
|
862 | env_setting = env.get('NOSE_IPDOCTEST_EXTENSION') | |
|
863 | if env_setting is not None: | |
|
864 | parser.set_defaults(ipdoctest_extension=tolist(env_setting)) | |
|
840 | 865 | |
|
866 | def configure(self, options, config): | |
|
841 | 867 | Plugin.configure(self, options, config) |
|
842 | self.doctest_tests = options.doctest_tests | |
|
843 |
self.extension = tolist(options.doctest |
|
|
868 | self.doctest_tests = options.ipdoctest_tests | |
|
869 | self.extension = tolist(options.ipdoctest_extension) | |
|
844 | 870 | |
|
845 | 871 | self.parser = IPDocTestParser() |
|
846 | 872 | self.finder = DocTestFinder(parser=self.parser) |
|
847 | 873 | self.checker = IPDoctestOutputChecker() |
|
848 | 874 | self.globs = None |
|
849 | 875 | self.extraglobs = None |
|
850 |
@@ -6,8 +6,9 b' This is used by a companion test case.' | |||
|
6 | 6 | import gc |
|
7 | 7 | |
|
8 | 8 | class C(object): |
|
9 | def __del__(self): | |
|
10 | print 'deleting object...' | |
|
9 | def __del__(self): | |
|
10 | pass | |
|
11 | #print 'deleting object...' # dbg | |
|
11 | 12 | |
|
12 | 13 | c = C() |
|
13 | 14 |
@@ -39,13 +39,10 b' def doctest_ivars():' | |||
|
39 | 39 | Out[6]: 1 |
|
40 | 40 | """ |
|
41 | 41 | |
|
42 | @dec.skip_doctest | |
|
42 | #@dec.skip_doctest | |
|
43 | 43 | def doctest_refs(): |
|
44 | 44 | """DocTest reference holding issues when running scripts. |
|
45 | 45 | |
|
46 | 46 | In [32]: run show_refs.py |
|
47 | 47 | c referrers: [<type 'dict'>] |
|
48 | ||
|
49 | In [33]: map(type,gc.get_referrers(c)) | |
|
50 | Out[33]: [<type 'dict'>] | |
|
51 | 48 | """ |
@@ -26,7 +26,7 b' import sys' | |||
|
26 | 26 | |
|
27 | 27 | class A(object): |
|
28 | 28 | def __del__(self): |
|
29 | print 'object A deleted' | |
|
29 | print 'obj_del.py: object A deleted' | |
|
30 | 30 | |
|
31 | 31 | a = A() |
|
32 | 32 |
@@ -16,11 +16,12 b' class C(object):' | |||
|
16 | 16 | self.name = name |
|
17 | 17 | |
|
18 | 18 | def __del__(self): |
|
19 |
print ' |
|
|
19 | print 'tclass.py: deleting object:',self.name | |
|
20 | 20 | |
|
21 | 21 | try: |
|
22 | 22 | name = sys.argv[1] |
|
23 | 23 | except IndexError: |
|
24 | 24 | pass |
|
25 | 25 | else: |
|
26 | c = C(name) | |
|
26 | if name.startswith('C'): | |
|
27 | c = C(name) |
@@ -1,17 +1,68 b'' | |||
|
1 | 1 | """Tests for the key iplib module, where the main ipython class is defined. |
|
2 | 2 | """ |
|
3 | #----------------------------------------------------------------------------- | |
|
4 | # Module imports | |
|
5 | #----------------------------------------------------------------------------- | |
|
3 | 6 | |
|
7 | # stdlib | |
|
8 | import os | |
|
9 | import shutil | |
|
10 | import tempfile | |
|
11 | ||
|
12 | # third party | |
|
4 | 13 | import nose.tools as nt |
|
5 | 14 | |
|
15 | # our own packages | |
|
16 | from IPython import iplib | |
|
17 | ||
|
18 | #----------------------------------------------------------------------------- | |
|
19 | # Globals | |
|
20 | #----------------------------------------------------------------------------- | |
|
21 | ||
|
22 | # Useful global ipapi object and main IPython one. Unfortunately we have a | |
|
23 | # long precedent of carrying the 'ipapi' global object which is injected into | |
|
24 | # the system namespace as _ip, but that keeps a pointer to the actual IPython | |
|
25 | # InteractiveShell instance, which is named IP. Since in testing we do need | |
|
26 | # access to the real thing (we want to probe beyond what ipapi exposes), make | |
|
27 | # here a global reference to each. In general, things that are exposed by the | |
|
28 | # ipapi instance should be read from there, but we also will often need to use | |
|
29 | # the actual IPython one. | |
|
30 | ||
|
31 | ip = _ip # This is the ipapi instance | |
|
32 | IP = ip.IP # This is the actual IPython shell 'raw' object. | |
|
33 | ||
|
34 | #----------------------------------------------------------------------------- | |
|
35 | # Test functions | |
|
36 | #----------------------------------------------------------------------------- | |
|
6 | 37 | |
|
7 | 38 | def test_reset(): |
|
8 | 39 | """reset must clear most namespaces.""" |
|
9 | ip = _ip.IP | |
|
10 | ip.reset() # first, it should run without error | |
|
40 | IP.reset() # first, it should run without error | |
|
11 | 41 | # Then, check that most namespaces end up empty |
|
12 |
for ns in |
|
|
13 |
if ns is |
|
|
42 | for ns in IP.ns_refs_table: | |
|
43 | if ns is IP.user_ns: | |
|
14 | 44 | # The user namespace is reset with some data, so we can't check for |
|
15 | 45 | # it being empty |
|
16 | 46 | continue |
|
17 | 47 | nt.assert_equals(len(ns),0) |
|
48 | ||
|
49 | ||
|
50 | # make sure that user_setup can be run re-entrantly in 'install' mode. | |
|
51 | def test_user_setup(): | |
|
52 | # use a lambda to pass kwargs to the generator | |
|
53 | user_setup = lambda a,k: iplib.user_setup(*a,**k) | |
|
54 | kw = dict(mode='install', interactive=False) | |
|
55 | ||
|
56 | # Call the user setup and verify that the directory exists | |
|
57 | yield user_setup, (ip.options.ipythondir,''), kw | |
|
58 | yield os.path.isdir, ip.options.ipythondir | |
|
59 | ||
|
60 | # Now repeat the operation with a non-existent directory. Check both that | |
|
61 | # the call succeeds and that the directory is created. | |
|
62 | tmpdir = tempfile.mktemp(prefix='ipython-test-') | |
|
63 | try: | |
|
64 | yield user_setup, (tmpdir,''), kw | |
|
65 | yield os.path.isdir, tmpdir | |
|
66 | finally: | |
|
67 | # In this case, clean up the temp dir once done | |
|
68 | shutil.rmtree(tmpdir) |
@@ -37,7 +37,7 b' def test_rehashx():' | |||
|
37 | 37 | def doctest_run_ns(): |
|
38 | 38 | """Classes declared %run scripts must be instantiable afterwards. |
|
39 | 39 | |
|
40 | In [11]: run tclass | |
|
40 | In [11]: run tclass foo | |
|
41 | 41 | |
|
42 | 42 | In [12]: isinstance(f(),foo) |
|
43 | 43 | Out[12]: True |
@@ -47,12 +47,10 b' def doctest_run_ns():' | |||
|
47 | 47 | def doctest_run_ns2(): |
|
48 | 48 | """Classes declared %run scripts must be instantiable afterwards. |
|
49 | 49 | |
|
50 |
In [ |
|
|
50 | In [4]: run tclass C-first_pass | |
|
51 | 51 | |
|
52 |
In [ |
|
|
53 | ||
|
54 | In [5]: run tclass second_pass | |
|
55 | Deleting object: first_pass | |
|
52 | In [5]: run tclass C-second_pass | |
|
53 | tclass.py: deleting object: C-first_pass | |
|
56 | 54 | """ |
|
57 | 55 | |
|
58 | 56 | |
@@ -85,7 +83,7 b' def test_obj_del():' | |||
|
85 | 83 | test_dir = os.path.dirname(__file__) |
|
86 | 84 | del_file = os.path.join(test_dir,'obj_del.py') |
|
87 | 85 | out = _ip.IP.getoutput('ipython %s' % del_file) |
|
88 | nt.assert_equals(out,'object A deleted') | |
|
86 | nt.assert_equals(out,'obj_del.py: object A deleted') | |
|
89 | 87 | |
|
90 | 88 | |
|
91 | 89 | def test_shist(): |
@@ -133,3 +131,21 b' def test_fail_dec2(*a,**k):' | |||
|
133 | 131 | def test_fail_dec3(*a,**k): |
|
134 | 132 | yield nt.assert_true, False |
|
135 | 133 | |
|
134 | ||
|
135 | def doctest_refbug(): | |
|
136 | """Very nasty problem with references held by multiple runs of a script. | |
|
137 | See: https://bugs.launchpad.net/ipython/+bug/269966 | |
|
138 | ||
|
139 | In [1]: _ip.IP.clear_main_mod_cache() | |
|
140 | ||
|
141 | In [2]: run refbug | |
|
142 | ||
|
143 | In [3]: call_f() | |
|
144 | lowercased: hello | |
|
145 | ||
|
146 | In [4]: run refbug | |
|
147 | ||
|
148 | In [5]: call_f() | |
|
149 | lowercased: hello | |
|
150 | lowercased: hello | |
|
151 | """ |
@@ -75,6 +75,9 b' Bug fixes' | |||
|
75 | 75 | The block is assigned to pasted_block even if code |
|
76 | 76 | raises exception. |
|
77 | 77 | |
|
78 | * Bug #274067 'The code in get_home_dir is broken for py2exe' was | |
|
79 | fixed. | |
|
80 | ||
|
78 | 81 | Backwards incompatible changes |
|
79 | 82 | ------------------------------ |
|
80 | 83 |
@@ -345,6 +345,37 b' nosetests option. For example, you can use ``--pdb`` or ``--pdb-failures`` to' | |||
|
345 | 345 | automatically activate the interactive Pdb debugger on errors or failures. See |
|
346 | 346 | the nosetests documentation for further details. |
|
347 | 347 | |
|
348 | .. warning:: | |
|
349 | ||
|
350 | Note that right now we have a nasty interaction between ipdoctest and | |
|
351 | twisted. Until we figure this out, please use the following instructions to | |
|
352 | ensure that at least you run all the tests. | |
|
353 | ||
|
354 | Right now, if you now run:: | |
|
355 | ||
|
356 | $ iptest [any options] [any submodules] | |
|
357 | ||
|
358 | it will NOT load ipdoctest but won't cause any Twisted problems. | |
|
359 | ||
|
360 | Once you're happy that you didn't break Twisted, run:: | |
|
361 | ||
|
362 | $ iptest --with-ipdoctest [any options] [any submodules] | |
|
363 | ||
|
364 | This MAY give a Twisted AlreadyCalledError exception at the end, but it will | |
|
365 | also correctly load up all of the ipython-specific tests and doctests. | |
|
366 | ||
|
367 | The above can be made easier with a trivial shell alias:: | |
|
368 | ||
|
369 | $ alias iptest2='iptest --with-ipdoctest' | |
|
370 | ||
|
371 | So that you can run:: | |
|
372 | ||
|
373 | $ iptest ... | |
|
374 | # Twisted happy | |
|
375 | # iptest2 ... | |
|
376 | # ignore possible Twisted error, this checks all the rest. | |
|
377 | ||
|
378 | ||
|
348 | 379 | A few tips for writing tests |
|
349 | 380 | ---------------------------- |
|
350 | 381 |
@@ -32,34 +32,34 b' Starting the engines with MPI enabled' | |||
|
32 | 32 | To use code that calls MPI, there are typically two things that MPI requires. |
|
33 | 33 | |
|
34 | 34 | 1. The process that wants to call MPI must be started using |
|
35 |
:command:`mpi |
|
|
35 | :command:`mpiexec` or a batch system (like PBS) that has MPI support. | |
|
36 | 36 | 2. Once the process starts, it must call :func:`MPI_Init`. |
|
37 | 37 | |
|
38 | 38 | There are a couple of ways that you can start the IPython engines and get these things to happen. |
|
39 | 39 | |
|
40 |
Automatic starting using :command:`mpi |
|
|
40 | Automatic starting using :command:`mpiexec` and :command:`ipcluster` | |
|
41 | 41 | ------------------------------------------------------------------- |
|
42 | 42 | |
|
43 |
The easiest approach is to use the `mpi |
|
|
43 | The easiest approach is to use the `mpiexec` mode of :command:`ipcluster`, which will first start a controller and then a set of engines using :command:`mpiexec`:: | |
|
44 | 44 | |
|
45 |
$ ipcluster mpi |
|
|
45 | $ ipcluster mpiexec -n 4 | |
|
46 | 46 | |
|
47 | 47 | This approach is best as interrupting :command:`ipcluster` will automatically |
|
48 | 48 | stop and clean up the controller and engines. |
|
49 | 49 | |
|
50 |
Manual starting using :command:`mpi |
|
|
50 | Manual starting using :command:`mpiexec` | |
|
51 | 51 | --------------------------------------- |
|
52 | 52 | |
|
53 |
If you want to start the IPython engines using the :command:`mpi |
|
|
53 | If you want to start the IPython engines using the :command:`mpiexec`, just do:: | |
|
54 | 54 | |
|
55 |
$ mpi |
|
|
55 | $ mpiexec -n 4 ipengine --mpi=mpi4py | |
|
56 | 56 | |
|
57 | 57 | This requires that you already have a controller running and that the FURL |
|
58 | 58 | files for the engines are in place. We also have built in support for |
|
59 | 59 | PyTrilinos [PyTrilinos]_, which can be used (assuming is installed) by |
|
60 | 60 | starting the engines with:: |
|
61 | 61 | |
|
62 |
mpi |
|
|
62 | mpiexec -n 4 ipengine --mpi=pytrilinos | |
|
63 | 63 | |
|
64 | 64 | Automatic starting using PBS and :command:`ipcluster` |
|
65 | 65 | ----------------------------------------------------- |
@@ -84,7 +84,7 b' First, lets define a simply function that uses MPI to calculate the sum of a dis' | |||
|
84 | 84 | |
|
85 | 85 | Now, start an IPython cluster in the same directory as :file:`psum.py`:: |
|
86 | 86 | |
|
87 |
$ ipcluster mpi |
|
|
87 | $ ipcluster mpiexec -n 4 | |
|
88 | 88 | |
|
89 | 89 | Finally, connect to the cluster and use this function interactively. In this case, we create a random array on each engine and sum up all the random arrays using our :func:`psum` function: |
|
90 | 90 |
@@ -85,33 +85,40 b' To see other command line options for the local mode, do::' | |||
|
85 | 85 | |
|
86 | 86 | $ ipcluster local -h |
|
87 | 87 | |
|
88 | Using :command:`ipcluster` in mpirun mode | |
|
89 | ----------------------------------------- | |
|
88 | Using :command:`ipcluster` in mpiexec/mpirun mode | |
|
89 | ------------------------------------------------- | |
|
90 | 90 | |
|
91 | The mpirun mode is useful if you: | |
|
91 | The mpiexec/mpirun mode is useful if you: | |
|
92 | 92 | |
|
93 | 93 | 1. Have MPI installed. |
|
94 |
2. Your systems are configured to use the :command:`mpi |
|
|
95 | processes. | |
|
94 | 2. Your systems are configured to use the :command:`mpiexec` or | |
|
95 | :command:`mpirun` commands to start MPI processes. | |
|
96 | ||
|
97 | .. note:: | |
|
98 | ||
|
99 | The preferred command to use is :command:`mpiexec`. However, we also | |
|
100 | support :command:`mpirun` for backwards compatibility. The underlying | |
|
101 | logic used is exactly the same, the only difference being the name of the | |
|
102 | command line program that is called. | |
|
96 | 103 | |
|
97 | 104 | If these are satisfied, you can start an IPython cluster using:: |
|
98 | 105 | |
|
99 |
$ ipcluster mpi |
|
|
106 | $ ipcluster mpiexec -n 4 | |
|
100 | 107 | |
|
101 | 108 | This does the following: |
|
102 | 109 | |
|
103 | 110 | 1. Starts the IPython controller on current host. |
|
104 |
2. Uses :command:`mpi |
|
|
111 | 2. Uses :command:`mpiexec` to start 4 engines. | |
|
105 | 112 | |
|
106 | 113 | On newer MPI implementations (such as OpenMPI), this will work even if you don't make any calls to MPI or call :func:`MPI_Init`. However, older MPI implementations actually require each process to call :func:`MPI_Init` upon starting. The easiest way of having this done is to install the mpi4py [mpi4py]_ package and then call ipcluster with the ``--mpi`` option:: |
|
107 | 114 | |
|
108 |
$ ipcluster mpi |
|
|
115 | $ ipcluster mpiexec -n 4 --mpi=mpi4py | |
|
109 | 116 | |
|
110 | 117 | Unfortunately, even this won't work for some MPI implementations. If you are having problems with this, you will likely have to use a custom Python executable that itself calls :func:`MPI_Init` at the appropriate time. Fortunately, mpi4py comes with such a custom Python executable that is easy to install and use. However, this custom Python executable approach will not work with :command:`ipcluster` currently. |
|
111 | 118 | |
|
112 | 119 | Additional command line options for this mode can be found by doing:: |
|
113 | 120 | |
|
114 |
$ ipcluster mpi |
|
|
121 | $ ipcluster mpiexec -h | |
|
115 | 122 | |
|
116 | 123 | More details on using MPI with IPython can be found :ref:`here <parallelmpi>`. |
|
117 | 124 | |
@@ -301,6 +308,11 b' This is possible. The only thing you have to do is decide what ports the contro' | |||
|
301 | 308 | |
|
302 | 309 | $ ipcontroller -r --client-port=10101 --engine-port=10102 |
|
303 | 310 | |
|
311 | These options also work with all of the various modes of | |
|
312 | :command:`ipcluster`:: | |
|
313 | ||
|
314 | $ ipcluster local -n 2 -r --client-port=10101 --engine-port=10102 | |
|
315 | ||
|
304 | 316 | Then, just copy the furl files over the first time and you are set. You can start and stop the controller and engines any many times as you want in the future, just make sure to tell the controller to use the *same* ports. |
|
305 | 317 | |
|
306 | 318 | .. note:: |
@@ -5,9 +5,15 b'' | |||
|
5 | 5 | |
|
6 | 6 | from docutils.nodes import Body, Element |
|
7 | 7 | from docutils.writers.html4css1 import HTMLTranslator |
|
8 | from sphinx.latexwriter import LaTeXTranslator | |
|
9 | 8 | from docutils.parsers.rst import directives |
|
10 | 9 | |
|
10 | # The sphinx API has changed, so we try both the old and new import forms | |
|
11 | try: | |
|
12 | from sphinx.latexwriter import LaTeXTranslator | |
|
13 | except ImportError: | |
|
14 | from sphinx.writers.latex import LaTeXTranslator | |
|
15 | ||
|
16 | ||
|
11 | 17 | class html_only(Body, Element): |
|
12 | 18 | pass |
|
13 | 19 |
General Comments 0
You need to be logged in to leave comments.
Login now