##// END OF EJS Templates
Fixing installation related issues.
Brian Granger -
Show More
@@ -1,252 +1,252 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 Test process execution and IO redirection.
3 Test process execution and IO redirection.
4 """
4 """
5
5
6 __docformat__ = "restructuredtext en"
6 __docformat__ = "restructuredtext en"
7
7
8 #-------------------------------------------------------------------------------
8 #-------------------------------------------------------------------------------
9 # Copyright (C) 2008 The IPython Development Team
9 # Copyright (C) 2008 The IPython Development Team
10 #
10 #
11 # Distributed under the terms of the BSD License. The full license is
11 # Distributed under the terms of the BSD License. The full license is
12 # in the file COPYING, distributed as part of this software.
12 # in the file COPYING, distributed as part of this software.
13 #-------------------------------------------------------------------------------
13 #-------------------------------------------------------------------------------
14
14
15 from copy import copy, deepcopy
15 from copy import copy, deepcopy
16 from cStringIO import StringIO
16 from cStringIO import StringIO
17 import string
17 import string
18
18
19 from nose.tools import assert_equal
19 from nose.tools import assert_equal
20
20
21 from IPython.frontend.prefilterfrontend import PrefilterFrontEnd
21 from IPython.frontend.prefilterfrontend import PrefilterFrontEnd
22 from IPython.core.ipapi import get as get_ipython0
22 from IPython.core.ipapi import get as get_ipython0
23 from IPython.testing.plugin.ipdoctest import default_argv
23 from IPython.testing.plugin.ipdoctest import default_argv
24
24
25
25
26 def safe_deepcopy(d):
26 def safe_deepcopy(d):
27 """ Deep copy every key of the given dict, when possible. Elsewhere
27 """ Deep copy every key of the given dict, when possible. Elsewhere
28 do a copy.
28 do a copy.
29 """
29 """
30 copied_d = dict()
30 copied_d = dict()
31 for key, value in d.iteritems():
31 for key, value in d.iteritems():
32 try:
32 try:
33 copied_d[key] = deepcopy(value)
33 copied_d[key] = deepcopy(value)
34 except:
34 except:
35 try:
35 try:
36 copied_d[key] = copy(value)
36 copied_d[key] = copy(value)
37 except:
37 except:
38 copied_d[key] = value
38 copied_d[key] = value
39 return copied_d
39 return copied_d
40
40
41
41
42 class TestPrefilterFrontEnd(PrefilterFrontEnd):
42 class TestPrefilterFrontEnd(PrefilterFrontEnd):
43
43
44 input_prompt_template = string.Template('')
44 input_prompt_template = string.Template('')
45 output_prompt_template = string.Template('')
45 output_prompt_template = string.Template('')
46 banner = ''
46 banner = ''
47
47
48 def __init__(self):
48 def __init__(self):
49 self.out = StringIO()
49 self.out = StringIO()
50 PrefilterFrontEnd.__init__(self,argv=default_argv())
50 PrefilterFrontEnd.__init__(self,argv=default_argv())
51 # Some more code for isolation (yeah, crazy)
51 # Some more code for isolation (yeah, crazy)
52 self._on_enter()
52 self._on_enter()
53 self.out.flush()
53 self.out.flush()
54 self.out.reset()
54 self.out.reset()
55 self.out.truncate()
55 self.out.truncate()
56
56
57 def write(self, string, *args, **kwargs):
57 def write(self, string, *args, **kwargs):
58 self.out.write(string)
58 self.out.write(string)
59
59
60 def _on_enter(self):
60 def _on_enter(self):
61 self.input_buffer += '\n'
61 self.input_buffer += '\n'
62 PrefilterFrontEnd._on_enter(self)
62 PrefilterFrontEnd._on_enter(self)
63
63
64
64
65 def isolate_ipython0(func):
65 def isolate_ipython0(func):
66 """ Decorator to isolate execution that involves an iptyhon0.
66 """ Decorator to isolate execution that involves an iptyhon0.
67
67
68 Notes
68 Notes
69 -----
69 -----
70
70
71 Apply only to functions with no arguments. Nose skips functions
71 Apply only to functions with no arguments. Nose skips functions
72 with arguments.
72 with arguments.
73 """
73 """
74 def my_func():
74 def my_func():
75 iplib = get_ipython0()
75 iplib = get_ipython0()
76 if iplib is None:
76 if iplib is None:
77 return func()
77 return func()
78 ipython0 = iplib.IP
78 ipython0 = iplib.IP
79 global_ns = safe_deepcopy(ipython0.user_global_ns)
79 global_ns = safe_deepcopy(ipython0.user_global_ns)
80 user_ns = safe_deepcopy(ipython0.user_ns)
80 user_ns = safe_deepcopy(ipython0.user_ns)
81 try:
81 try:
82 out = func()
82 out = func()
83 finally:
83 finally:
84 ipython0.user_ns = user_ns
84 ipython0.user_ns = user_ns
85 ipython0.user_global_ns = global_ns
85 ipython0.user_global_ns = global_ns
86 # Undo the hack at creation of PrefilterFrontEnd
86 # Undo the hack at creation of PrefilterFrontEnd
87 from IPythoncore. import iplib
87 from IPython.core import iplib
88 iplib.InteractiveShell.isthreaded = False
88 iplib.InteractiveShell.isthreaded = False
89 return out
89 return out
90
90
91 my_func.__name__ = func.__name__
91 my_func.__name__ = func.__name__
92 return my_func
92 return my_func
93
93
94
94
95 @isolate_ipython0
95 @isolate_ipython0
96 def test_execution():
96 def test_execution():
97 """ Test execution of a command.
97 """ Test execution of a command.
98 """
98 """
99 f = TestPrefilterFrontEnd()
99 f = TestPrefilterFrontEnd()
100 f.input_buffer = 'print 1'
100 f.input_buffer = 'print 1'
101 f._on_enter()
101 f._on_enter()
102 out_value = f.out.getvalue()
102 out_value = f.out.getvalue()
103 assert_equal(out_value, '1\n')
103 assert_equal(out_value, '1\n')
104
104
105
105
106 @isolate_ipython0
106 @isolate_ipython0
107 def test_multiline():
107 def test_multiline():
108 """ Test execution of a multiline command.
108 """ Test execution of a multiline command.
109 """
109 """
110 f = TestPrefilterFrontEnd()
110 f = TestPrefilterFrontEnd()
111 f.input_buffer = 'if True:'
111 f.input_buffer = 'if True:'
112 f._on_enter()
112 f._on_enter()
113 f.input_buffer += 'print 1'
113 f.input_buffer += 'print 1'
114 f._on_enter()
114 f._on_enter()
115 out_value = f.out.getvalue()
115 out_value = f.out.getvalue()
116 yield assert_equal, out_value, ''
116 yield assert_equal, out_value, ''
117 f._on_enter()
117 f._on_enter()
118 out_value = f.out.getvalue()
118 out_value = f.out.getvalue()
119 yield assert_equal, out_value, '1\n'
119 yield assert_equal, out_value, '1\n'
120 f = TestPrefilterFrontEnd()
120 f = TestPrefilterFrontEnd()
121 f.input_buffer='(1 +'
121 f.input_buffer='(1 +'
122 f._on_enter()
122 f._on_enter()
123 f.input_buffer += '0)'
123 f.input_buffer += '0)'
124 f._on_enter()
124 f._on_enter()
125 out_value = f.out.getvalue()
125 out_value = f.out.getvalue()
126 yield assert_equal, out_value, ''
126 yield assert_equal, out_value, ''
127 f._on_enter()
127 f._on_enter()
128 out_value = f.out.getvalue()
128 out_value = f.out.getvalue()
129 yield assert_equal, out_value, '1\n'
129 yield assert_equal, out_value, '1\n'
130
130
131
131
132 @isolate_ipython0
132 @isolate_ipython0
133 def test_capture():
133 def test_capture():
134 """ Test the capture of output in different channels.
134 """ Test the capture of output in different channels.
135 """
135 """
136 # Test on the OS-level stdout, stderr.
136 # Test on the OS-level stdout, stderr.
137 f = TestPrefilterFrontEnd()
137 f = TestPrefilterFrontEnd()
138 f.input_buffer = \
138 f.input_buffer = \
139 'import os; out=os.fdopen(1, "w"); out.write("1") ; out.flush()'
139 'import os; out=os.fdopen(1, "w"); out.write("1") ; out.flush()'
140 f._on_enter()
140 f._on_enter()
141 out_value = f.out.getvalue()
141 out_value = f.out.getvalue()
142 yield assert_equal, out_value, '1'
142 yield assert_equal, out_value, '1'
143 f = TestPrefilterFrontEnd()
143 f = TestPrefilterFrontEnd()
144 f.input_buffer = \
144 f.input_buffer = \
145 'import os; out=os.fdopen(2, "w"); out.write("1") ; out.flush()'
145 'import os; out=os.fdopen(2, "w"); out.write("1") ; out.flush()'
146 f._on_enter()
146 f._on_enter()
147 out_value = f.out.getvalue()
147 out_value = f.out.getvalue()
148 yield assert_equal, out_value, '1'
148 yield assert_equal, out_value, '1'
149
149
150
150
151 @isolate_ipython0
151 @isolate_ipython0
152 def test_magic():
152 def test_magic():
153 """ Test the magic expansion and history.
153 """ Test the magic expansion and history.
154
154
155 This test is fairly fragile and will break when magics change.
155 This test is fairly fragile and will break when magics change.
156 """
156 """
157 f = TestPrefilterFrontEnd()
157 f = TestPrefilterFrontEnd()
158 # Before checking the interactive namespace, make sure it's clear (it can
158 # Before checking the interactive namespace, make sure it's clear (it can
159 # otherwise pick up things stored in the user's local db)
159 # otherwise pick up things stored in the user's local db)
160 f.input_buffer += '%reset -f'
160 f.input_buffer += '%reset -f'
161 f._on_enter()
161 f._on_enter()
162 f.complete_current_input()
162 f.complete_current_input()
163 # Now, run the %who magic and check output
163 # Now, run the %who magic and check output
164 f.input_buffer += '%who'
164 f.input_buffer += '%who'
165 f._on_enter()
165 f._on_enter()
166 out_value = f.out.getvalue()
166 out_value = f.out.getvalue()
167 assert_equal(out_value, 'Interactive namespace is empty.\n')
167 assert_equal(out_value, 'Interactive namespace is empty.\n')
168
168
169
169
170 @isolate_ipython0
170 @isolate_ipython0
171 def test_help():
171 def test_help():
172 """ Test object inspection.
172 """ Test object inspection.
173 """
173 """
174 f = TestPrefilterFrontEnd()
174 f = TestPrefilterFrontEnd()
175 f.input_buffer += "def f():"
175 f.input_buffer += "def f():"
176 f._on_enter()
176 f._on_enter()
177 f.input_buffer += "'foobar'"
177 f.input_buffer += "'foobar'"
178 f._on_enter()
178 f._on_enter()
179 f.input_buffer += "pass"
179 f.input_buffer += "pass"
180 f._on_enter()
180 f._on_enter()
181 f._on_enter()
181 f._on_enter()
182 f.input_buffer += "f?"
182 f.input_buffer += "f?"
183 f._on_enter()
183 f._on_enter()
184 assert 'traceback' not in f.last_result
184 assert 'traceback' not in f.last_result
185 ## XXX: ipython doctest magic breaks this. I have no clue why
185 ## XXX: ipython doctest magic breaks this. I have no clue why
186 #out_value = f.out.getvalue()
186 #out_value = f.out.getvalue()
187 #assert out_value.split()[-1] == 'foobar'
187 #assert out_value.split()[-1] == 'foobar'
188
188
189
189
190 @isolate_ipython0
190 @isolate_ipython0
191 def test_completion_simple():
191 def test_completion_simple():
192 """ Test command-line completion on trivial examples.
192 """ Test command-line completion on trivial examples.
193 """
193 """
194 f = TestPrefilterFrontEnd()
194 f = TestPrefilterFrontEnd()
195 f.input_buffer = 'zzza = 1'
195 f.input_buffer = 'zzza = 1'
196 f._on_enter()
196 f._on_enter()
197 f.input_buffer = 'zzzb = 2'
197 f.input_buffer = 'zzzb = 2'
198 f._on_enter()
198 f._on_enter()
199 f.input_buffer = 'zz'
199 f.input_buffer = 'zz'
200 f.complete_current_input()
200 f.complete_current_input()
201 out_value = f.out.getvalue()
201 out_value = f.out.getvalue()
202 yield assert_equal, out_value, '\nzzza zzzb '
202 yield assert_equal, out_value, '\nzzza zzzb '
203 yield assert_equal, f.input_buffer, 'zzz'
203 yield assert_equal, f.input_buffer, 'zzz'
204
204
205
205
206 @isolate_ipython0
206 @isolate_ipython0
207 def test_completion_parenthesis():
207 def test_completion_parenthesis():
208 """ Test command-line completion when a parenthesis is open.
208 """ Test command-line completion when a parenthesis is open.
209 """
209 """
210 f = TestPrefilterFrontEnd()
210 f = TestPrefilterFrontEnd()
211 f.input_buffer = 'zzza = 1'
211 f.input_buffer = 'zzza = 1'
212 f._on_enter()
212 f._on_enter()
213 f.input_buffer = 'zzzb = 2'
213 f.input_buffer = 'zzzb = 2'
214 f._on_enter()
214 f._on_enter()
215 f.input_buffer = 'map(zz'
215 f.input_buffer = 'map(zz'
216 f.complete_current_input()
216 f.complete_current_input()
217 out_value = f.out.getvalue()
217 out_value = f.out.getvalue()
218 yield assert_equal, out_value, '\nzzza zzzb '
218 yield assert_equal, out_value, '\nzzza zzzb '
219 yield assert_equal, f.input_buffer, 'map(zzz'
219 yield assert_equal, f.input_buffer, 'map(zzz'
220
220
221
221
222 @isolate_ipython0
222 @isolate_ipython0
223 def test_completion_indexing():
223 def test_completion_indexing():
224 """ Test command-line completion when indexing on objects.
224 """ Test command-line completion when indexing on objects.
225 """
225 """
226 f = TestPrefilterFrontEnd()
226 f = TestPrefilterFrontEnd()
227 f.input_buffer = 'a = [0]'
227 f.input_buffer = 'a = [0]'
228 f._on_enter()
228 f._on_enter()
229 f.input_buffer = 'a[0].'
229 f.input_buffer = 'a[0].'
230 f.complete_current_input()
230 f.complete_current_input()
231 assert_equal(f.input_buffer, 'a[0].__')
231 assert_equal(f.input_buffer, 'a[0].__')
232
232
233
233
234 @isolate_ipython0
234 @isolate_ipython0
235 def test_completion_equal():
235 def test_completion_equal():
236 """ Test command-line completion when the delimiter is "=", not " ".
236 """ Test command-line completion when the delimiter is "=", not " ".
237 """
237 """
238 f = TestPrefilterFrontEnd()
238 f = TestPrefilterFrontEnd()
239 f.input_buffer = 'a=1.'
239 f.input_buffer = 'a=1.'
240 f.complete_current_input()
240 f.complete_current_input()
241 assert_equal(f.input_buffer, 'a=1.__')
241 assert_equal(f.input_buffer, 'a=1.__')
242
242
243
243
244
244
245 if __name__ == '__main__':
245 if __name__ == '__main__':
246 test_magic()
246 test_magic()
247 test_help()
247 test_help()
248 test_execution()
248 test_execution()
249 test_multiline()
249 test_multiline()
250 test_capture()
250 test_capture()
251 test_completion_simple()
251 test_completion_simple()
252 test_completion_complex()
252 test_completion_complex()
@@ -1,96 +1,96 b''
1 # encoding: utf-8
1 # encoding: utf-8
2
2
3 """This module contains blocking clients for the controller interfaces.
3 """This module contains blocking clients for the controller interfaces.
4
4
5 Unlike the clients in `asyncclient.py`, the clients in this module are fully
5 Unlike the clients in `asyncclient.py`, the clients in this module are fully
6 blocking. This means that methods on the clients return the actual results
6 blocking. This means that methods on the clients return the actual results
7 rather than a deferred to the result. Also, we manage the Twisted reactor
7 rather than a deferred to the result. Also, we manage the Twisted reactor
8 for you. This is done by running the reactor in a thread.
8 for you. This is done by running the reactor in a thread.
9
9
10 The main classes in this module are:
10 The main classes in this module are:
11
11
12 * MultiEngineClient
12 * MultiEngineClient
13 * TaskClient
13 * TaskClient
14 * Task
14 * Task
15 * CompositeError
15 * CompositeError
16 """
16 """
17
17
18 __docformat__ = "restructuredtext en"
18 __docformat__ = "restructuredtext en"
19
19
20 #-------------------------------------------------------------------------------
20 #-------------------------------------------------------------------------------
21 # Copyright (C) 2008 The IPython Development Team
21 # Copyright (C) 2008 The IPython Development Team
22 #
22 #
23 # Distributed under the terms of the BSD License. The full license is in
23 # Distributed under the terms of the BSD License. The full license is in
24 # the file COPYING, distributed as part of this software.
24 # the file COPYING, distributed as part of this software.
25 #-------------------------------------------------------------------------------
25 #-------------------------------------------------------------------------------
26
26
27 #-------------------------------------------------------------------------------
27 #-------------------------------------------------------------------------------
28 # Imports
28 # Imports
29 #-------------------------------------------------------------------------------
29 #-------------------------------------------------------------------------------
30
30
31 import sys
31 import sys
32
32
33 # from IPython.tools import growl
33 # from IPython.utils import growl
34 # growl.start("IPython1 Client")
34 # growl.start("IPython1 Client")
35
35
36
36
37 from twisted.internet import reactor
37 from twisted.internet import reactor
38 from IPython.kernel.clientconnector import ClientConnector
38 from IPython.kernel.clientconnector import ClientConnector
39 from IPython.kernel.twistedutil import ReactorInThread
39 from IPython.kernel.twistedutil import ReactorInThread
40 from IPython.kernel.twistedutil import blockingCallFromThread
40 from IPython.kernel.twistedutil import blockingCallFromThread
41
41
42 # These enable various things
42 # These enable various things
43 from IPython.kernel import codeutil
43 from IPython.kernel import codeutil
44 import IPython.kernel.magic
44 import IPython.kernel.magic
45
45
46 # Other things that the user will need
46 # Other things that the user will need
47 from IPython.kernel.task import MapTask, StringTask
47 from IPython.kernel.task import MapTask, StringTask
48 from IPython.kernel.error import CompositeError
48 from IPython.kernel.error import CompositeError
49
49
50 #-------------------------------------------------------------------------------
50 #-------------------------------------------------------------------------------
51 # Code
51 # Code
52 #-------------------------------------------------------------------------------
52 #-------------------------------------------------------------------------------
53
53
54 _client_tub = ClientConnector()
54 _client_tub = ClientConnector()
55
55
56
56
57 def get_multiengine_client(furl_or_file=''):
57 def get_multiengine_client(furl_or_file=''):
58 """Get the blocking MultiEngine client.
58 """Get the blocking MultiEngine client.
59
59
60 :Parameters:
60 :Parameters:
61 furl_or_file : str
61 furl_or_file : str
62 A furl or a filename containing a furl. If empty, the
62 A furl or a filename containing a furl. If empty, the
63 default furl_file will be used
63 default furl_file will be used
64
64
65 :Returns:
65 :Returns:
66 The connected MultiEngineClient instance
66 The connected MultiEngineClient instance
67 """
67 """
68 client = blockingCallFromThread(_client_tub.get_multiengine_client,
68 client = blockingCallFromThread(_client_tub.get_multiengine_client,
69 furl_or_file)
69 furl_or_file)
70 return client.adapt_to_blocking_client()
70 return client.adapt_to_blocking_client()
71
71
72 def get_task_client(furl_or_file=''):
72 def get_task_client(furl_or_file=''):
73 """Get the blocking Task client.
73 """Get the blocking Task client.
74
74
75 :Parameters:
75 :Parameters:
76 furl_or_file : str
76 furl_or_file : str
77 A furl or a filename containing a furl. If empty, the
77 A furl or a filename containing a furl. If empty, the
78 default furl_file will be used
78 default furl_file will be used
79
79
80 :Returns:
80 :Returns:
81 The connected TaskClient instance
81 The connected TaskClient instance
82 """
82 """
83 client = blockingCallFromThread(_client_tub.get_task_client,
83 client = blockingCallFromThread(_client_tub.get_task_client,
84 furl_or_file)
84 furl_or_file)
85 return client.adapt_to_blocking_client()
85 return client.adapt_to_blocking_client()
86
86
87
87
88 MultiEngineClient = get_multiengine_client
88 MultiEngineClient = get_multiengine_client
89 TaskClient = get_task_client
89 TaskClient = get_task_client
90
90
91
91
92
92
93 # Now we start the reactor in a thread
93 # Now we start the reactor in a thread
94 rit = ReactorInThread()
94 rit = ReactorInThread()
95 rit.setDaemon(True)
95 rit.setDaemon(True)
96 rit.start() No newline at end of file
96 rit.start()
@@ -1,53 +1,53 b''
1 # encoding: utf-8
1 # encoding: utf-8
2
2
3 """Object to manage sys.excepthook().
3 """Object to manage sys.excepthook().
4
4
5 Synchronous version: prints errors when called.
5 Synchronous version: prints errors when called.
6 """
6 """
7
7
8 __docformat__ = "restructuredtext en"
8 __docformat__ = "restructuredtext en"
9
9
10 #-------------------------------------------------------------------------------
10 #-------------------------------------------------------------------------------
11 # Copyright (C) 2008 The IPython Development Team
11 # Copyright (C) 2008 The IPython Development Team
12 #
12 #
13 # Distributed under the terms of the BSD License. The full license is in
13 # Distributed under the terms of the BSD License. The full license is in
14 # the file COPYING, distributed as part of this software.
14 # the file COPYING, distributed as part of this software.
15 #-------------------------------------------------------------------------------
15 #-------------------------------------------------------------------------------
16
16
17 #-------------------------------------------------------------------------------
17 #-------------------------------------------------------------------------------
18 # Imports
18 # Imports
19 #-------------------------------------------------------------------------------
19 #-------------------------------------------------------------------------------
20 from traceback_trap import TracebackTrap
20 from traceback_trap import TracebackTrap
21 from IPython.ultraTB import ColorTB
21 from IPython.core.ultratb import ColorTB
22
22
23 class SyncTracebackTrap(TracebackTrap):
23 class SyncTracebackTrap(TracebackTrap):
24 """ TracebackTrap that displays immediatly the traceback in addition
24 """ TracebackTrap that displays immediatly the traceback in addition
25 to capturing it. Useful in frontends, as without this traceback trap,
25 to capturing it. Useful in frontends, as without this traceback trap,
26 some tracebacks never get displayed.
26 some tracebacks never get displayed.
27 """
27 """
28
28
29 def __init__(self, sync_formatter=None, formatters=None,
29 def __init__(self, sync_formatter=None, formatters=None,
30 raiseException=True):
30 raiseException=True):
31 """
31 """
32 sync_formatter: Callable to display the traceback.
32 sync_formatter: Callable to display the traceback.
33 formatters: A list of formatters to apply.
33 formatters: A list of formatters to apply.
34 """
34 """
35 TracebackTrap.__init__(self, formatters=formatters)
35 TracebackTrap.__init__(self, formatters=formatters)
36 if sync_formatter is None:
36 if sync_formatter is None:
37 sync_formatter = ColorTB(color_scheme='LightBG')
37 sync_formatter = ColorTB(color_scheme='LightBG')
38 self.sync_formatter = sync_formatter
38 self.sync_formatter = sync_formatter
39 self.raiseException = raiseException
39 self.raiseException = raiseException
40
40
41
41
42 def hook(self, *args):
42 def hook(self, *args):
43 """ This method actually implements the hook.
43 """ This method actually implements the hook.
44 """
44 """
45 self.args = args
45 self.args = args
46 if not self.raiseException:
46 if not self.raiseException:
47 print self.sync_formatter(*self.args)
47 print self.sync_formatter(*self.args)
48 else:
48 else:
49 raise
49 raise
50
50
51
51
52
52
53
53
@@ -1,753 +1,753 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 # -*- test-case-name: IPython.kernel.test.test_multiengine -*-
2 # -*- test-case-name: IPython.kernel.test.test_multiengine -*-
3
3
4 """Adapt the IPython ControllerServer to IMultiEngine.
4 """Adapt the IPython ControllerServer to IMultiEngine.
5
5
6 This module provides classes that adapt a ControllerService to the
6 This module provides classes that adapt a ControllerService to the
7 IMultiEngine interface. This interface is a basic interactive interface
7 IMultiEngine interface. This interface is a basic interactive interface
8 for working with a set of engines where it is desired to have explicit
8 for working with a set of engines where it is desired to have explicit
9 access to each registered engine.
9 access to each registered engine.
10
10
11 The classes here are exposed to the network in files like:
11 The classes here are exposed to the network in files like:
12
12
13 * multienginevanilla.py
13 * multienginevanilla.py
14 * multienginepb.py
14 * multienginepb.py
15 """
15 """
16
16
17 __docformat__ = "restructuredtext en"
17 __docformat__ = "restructuredtext en"
18
18
19 #-------------------------------------------------------------------------------
19 #-------------------------------------------------------------------------------
20 # Copyright (C) 2008 The IPython Development Team
20 # Copyright (C) 2008 The IPython Development Team
21 #
21 #
22 # Distributed under the terms of the BSD License. The full license is in
22 # Distributed under the terms of the BSD License. The full license is in
23 # the file COPYING, distributed as part of this software.
23 # the file COPYING, distributed as part of this software.
24 #-------------------------------------------------------------------------------
24 #-------------------------------------------------------------------------------
25
25
26 #-------------------------------------------------------------------------------
26 #-------------------------------------------------------------------------------
27 # Imports
27 # Imports
28 #-------------------------------------------------------------------------------
28 #-------------------------------------------------------------------------------
29
29
30 from new import instancemethod
30 from new import instancemethod
31 from types import FunctionType
31 from types import FunctionType
32
32
33 from twisted.application import service
33 from twisted.application import service
34 from twisted.internet import defer, reactor
34 from twisted.internet import defer, reactor
35 from twisted.python import log, components, failure
35 from twisted.python import log, components, failure
36 from zope.interface import Interface, implements, Attribute
36 from zope.interface import Interface, implements, Attribute
37
37
38 from IPython.tools import growl
38 from IPython.utils import growl
39 from IPython.kernel.util import printer
39 from IPython.kernel.util import printer
40 from IPython.kernel.twistedutil import gatherBoth
40 from IPython.kernel.twistedutil import gatherBoth
41 from IPython.kernel import map as Map
41 from IPython.kernel import map as Map
42 from IPython.kernel import error
42 from IPython.kernel import error
43 from IPython.kernel.pendingdeferred import PendingDeferredManager, two_phase
43 from IPython.kernel.pendingdeferred import PendingDeferredManager, two_phase
44 from IPython.kernel.controllerservice import \
44 from IPython.kernel.controllerservice import \
45 ControllerAdapterBase, \
45 ControllerAdapterBase, \
46 ControllerService, \
46 ControllerService, \
47 IControllerBase
47 IControllerBase
48
48
49
49
50 #-------------------------------------------------------------------------------
50 #-------------------------------------------------------------------------------
51 # Interfaces for the MultiEngine representation of a controller
51 # Interfaces for the MultiEngine representation of a controller
52 #-------------------------------------------------------------------------------
52 #-------------------------------------------------------------------------------
53
53
54 class IEngineMultiplexer(Interface):
54 class IEngineMultiplexer(Interface):
55 """Interface to multiple engines implementing IEngineCore/Serialized/Queued.
55 """Interface to multiple engines implementing IEngineCore/Serialized/Queued.
56
56
57 This class simply acts as a multiplexer of methods that are in the
57 This class simply acts as a multiplexer of methods that are in the
58 various IEngines* interfaces. Thus the methods here are jut like those
58 various IEngines* interfaces. Thus the methods here are jut like those
59 in the IEngine* interfaces, but with an extra first argument, targets.
59 in the IEngine* interfaces, but with an extra first argument, targets.
60 The targets argument can have the following forms:
60 The targets argument can have the following forms:
61
61
62 * targets = 10 # Engines are indexed by ints
62 * targets = 10 # Engines are indexed by ints
63 * targets = [0,1,2,3] # A list of ints
63 * targets = [0,1,2,3] # A list of ints
64 * targets = 'all' # A string to indicate all targets
64 * targets = 'all' # A string to indicate all targets
65
65
66 If targets is bad in any way, an InvalidEngineID will be raised. This
66 If targets is bad in any way, an InvalidEngineID will be raised. This
67 includes engines not being registered.
67 includes engines not being registered.
68
68
69 All IEngineMultiplexer multiplexer methods must return a Deferred to a list
69 All IEngineMultiplexer multiplexer methods must return a Deferred to a list
70 with length equal to the number of targets. The elements of the list will
70 with length equal to the number of targets. The elements of the list will
71 correspond to the return of the corresponding IEngine method.
71 correspond to the return of the corresponding IEngine method.
72
72
73 Failures are aggressive, meaning that if an action fails for any target,
73 Failures are aggressive, meaning that if an action fails for any target,
74 the overall action will fail immediately with that Failure.
74 the overall action will fail immediately with that Failure.
75
75
76 :Parameters:
76 :Parameters:
77 targets : int, list of ints, or 'all'
77 targets : int, list of ints, or 'all'
78 Engine ids the action will apply to.
78 Engine ids the action will apply to.
79
79
80 :Returns: Deferred to a list of results for each engine.
80 :Returns: Deferred to a list of results for each engine.
81
81
82 :Exception:
82 :Exception:
83 InvalidEngineID
83 InvalidEngineID
84 If the targets argument is bad or engines aren't registered.
84 If the targets argument is bad or engines aren't registered.
85 NoEnginesRegistered
85 NoEnginesRegistered
86 If there are no engines registered and targets='all'
86 If there are no engines registered and targets='all'
87 """
87 """
88
88
89 #---------------------------------------------------------------------------
89 #---------------------------------------------------------------------------
90 # Mutiplexed methods
90 # Mutiplexed methods
91 #---------------------------------------------------------------------------
91 #---------------------------------------------------------------------------
92
92
93 def execute(lines, targets='all'):
93 def execute(lines, targets='all'):
94 """Execute lines of Python code on targets.
94 """Execute lines of Python code on targets.
95
95
96 See the class docstring for information about targets and possible
96 See the class docstring for information about targets and possible
97 exceptions this method can raise.
97 exceptions this method can raise.
98
98
99 :Parameters:
99 :Parameters:
100 lines : str
100 lines : str
101 String of python code to be executed on targets.
101 String of python code to be executed on targets.
102 """
102 """
103
103
104 def push(namespace, targets='all'):
104 def push(namespace, targets='all'):
105 """Push dict namespace into the user's namespace on targets.
105 """Push dict namespace into the user's namespace on targets.
106
106
107 See the class docstring for information about targets and possible
107 See the class docstring for information about targets and possible
108 exceptions this method can raise.
108 exceptions this method can raise.
109
109
110 :Parameters:
110 :Parameters:
111 namspace : dict
111 namspace : dict
112 Dict of key value pairs to be put into the users namspace.
112 Dict of key value pairs to be put into the users namspace.
113 """
113 """
114
114
115 def pull(keys, targets='all'):
115 def pull(keys, targets='all'):
116 """Pull values out of the user's namespace on targets by keys.
116 """Pull values out of the user's namespace on targets by keys.
117
117
118 See the class docstring for information about targets and possible
118 See the class docstring for information about targets and possible
119 exceptions this method can raise.
119 exceptions this method can raise.
120
120
121 :Parameters:
121 :Parameters:
122 keys : tuple of strings
122 keys : tuple of strings
123 Sequence of keys to be pulled from user's namespace.
123 Sequence of keys to be pulled from user's namespace.
124 """
124 """
125
125
126 def push_function(namespace, targets='all'):
126 def push_function(namespace, targets='all'):
127 """"""
127 """"""
128
128
129 def pull_function(keys, targets='all'):
129 def pull_function(keys, targets='all'):
130 """"""
130 """"""
131
131
132 def get_result(i=None, targets='all'):
132 def get_result(i=None, targets='all'):
133 """Get the result for command i from targets.
133 """Get the result for command i from targets.
134
134
135 See the class docstring for information about targets and possible
135 See the class docstring for information about targets and possible
136 exceptions this method can raise.
136 exceptions this method can raise.
137
137
138 :Parameters:
138 :Parameters:
139 i : int or None
139 i : int or None
140 Command index or None to indicate most recent command.
140 Command index or None to indicate most recent command.
141 """
141 """
142
142
143 def reset(targets='all'):
143 def reset(targets='all'):
144 """Reset targets.
144 """Reset targets.
145
145
146 This clears the users namespace of the Engines, but won't cause
146 This clears the users namespace of the Engines, but won't cause
147 modules to be reloaded.
147 modules to be reloaded.
148 """
148 """
149
149
150 def keys(targets='all'):
150 def keys(targets='all'):
151 """Get variable names defined in user's namespace on targets."""
151 """Get variable names defined in user's namespace on targets."""
152
152
153 def kill(controller=False, targets='all'):
153 def kill(controller=False, targets='all'):
154 """Kill the targets Engines and possibly the controller.
154 """Kill the targets Engines and possibly the controller.
155
155
156 :Parameters:
156 :Parameters:
157 controller : boolean
157 controller : boolean
158 Should the controller be killed as well. If so all the
158 Should the controller be killed as well. If so all the
159 engines will be killed first no matter what targets is.
159 engines will be killed first no matter what targets is.
160 """
160 """
161
161
162 def push_serialized(namespace, targets='all'):
162 def push_serialized(namespace, targets='all'):
163 """Push a namespace of Serialized objects to targets.
163 """Push a namespace of Serialized objects to targets.
164
164
165 :Parameters:
165 :Parameters:
166 namespace : dict
166 namespace : dict
167 A dict whose keys are the variable names and whose values
167 A dict whose keys are the variable names and whose values
168 are serialized version of the objects.
168 are serialized version of the objects.
169 """
169 """
170
170
171 def pull_serialized(keys, targets='all'):
171 def pull_serialized(keys, targets='all'):
172 """Pull Serialized objects by keys from targets.
172 """Pull Serialized objects by keys from targets.
173
173
174 :Parameters:
174 :Parameters:
175 keys : tuple of strings
175 keys : tuple of strings
176 Sequence of variable names to pull as serialized objects.
176 Sequence of variable names to pull as serialized objects.
177 """
177 """
178
178
179 def clear_queue(targets='all'):
179 def clear_queue(targets='all'):
180 """Clear the queue of pending command for targets."""
180 """Clear the queue of pending command for targets."""
181
181
182 def queue_status(targets='all'):
182 def queue_status(targets='all'):
183 """Get the status of the queue on the targets."""
183 """Get the status of the queue on the targets."""
184
184
185 def set_properties(properties, targets='all'):
185 def set_properties(properties, targets='all'):
186 """set properties by key and value"""
186 """set properties by key and value"""
187
187
188 def get_properties(keys=None, targets='all'):
188 def get_properties(keys=None, targets='all'):
189 """get a list of properties by `keys`, if no keys specified, get all"""
189 """get a list of properties by `keys`, if no keys specified, get all"""
190
190
191 def del_properties(keys, targets='all'):
191 def del_properties(keys, targets='all'):
192 """delete properties by `keys`"""
192 """delete properties by `keys`"""
193
193
194 def has_properties(keys, targets='all'):
194 def has_properties(keys, targets='all'):
195 """get a list of bool values for whether `properties` has `keys`"""
195 """get a list of bool values for whether `properties` has `keys`"""
196
196
197 def clear_properties(targets='all'):
197 def clear_properties(targets='all'):
198 """clear the properties dict"""
198 """clear the properties dict"""
199
199
200
200
201 class IMultiEngine(IEngineMultiplexer):
201 class IMultiEngine(IEngineMultiplexer):
202 """A controller that exposes an explicit interface to all of its engines.
202 """A controller that exposes an explicit interface to all of its engines.
203
203
204 This is the primary inteface for interactive usage.
204 This is the primary inteface for interactive usage.
205 """
205 """
206
206
207 def get_ids():
207 def get_ids():
208 """Return list of currently registered ids.
208 """Return list of currently registered ids.
209
209
210 :Returns: A Deferred to a list of registered engine ids.
210 :Returns: A Deferred to a list of registered engine ids.
211 """
211 """
212
212
213
213
214
214
215 #-------------------------------------------------------------------------------
215 #-------------------------------------------------------------------------------
216 # Implementation of the core MultiEngine classes
216 # Implementation of the core MultiEngine classes
217 #-------------------------------------------------------------------------------
217 #-------------------------------------------------------------------------------
218
218
219 class MultiEngine(ControllerAdapterBase):
219 class MultiEngine(ControllerAdapterBase):
220 """The representation of a ControllerService as a IMultiEngine.
220 """The representation of a ControllerService as a IMultiEngine.
221
221
222 Although it is not implemented currently, this class would be where a
222 Although it is not implemented currently, this class would be where a
223 client/notification API is implemented. It could inherit from something
223 client/notification API is implemented. It could inherit from something
224 like results.NotifierParent and then use the notify method to send
224 like results.NotifierParent and then use the notify method to send
225 notifications.
225 notifications.
226 """
226 """
227
227
228 implements(IMultiEngine)
228 implements(IMultiEngine)
229
229
230 def __init(self, controller):
230 def __init(self, controller):
231 ControllerAdapterBase.__init__(self, controller)
231 ControllerAdapterBase.__init__(self, controller)
232
232
233 #---------------------------------------------------------------------------
233 #---------------------------------------------------------------------------
234 # Helper methods
234 # Helper methods
235 #---------------------------------------------------------------------------
235 #---------------------------------------------------------------------------
236
236
237 def engineList(self, targets):
237 def engineList(self, targets):
238 """Parse the targets argument into a list of valid engine objects.
238 """Parse the targets argument into a list of valid engine objects.
239
239
240 :Parameters:
240 :Parameters:
241 targets : int, list of ints or 'all'
241 targets : int, list of ints or 'all'
242 The targets argument to be parsed.
242 The targets argument to be parsed.
243
243
244 :Returns: List of engine objects.
244 :Returns: List of engine objects.
245
245
246 :Exception:
246 :Exception:
247 InvalidEngineID
247 InvalidEngineID
248 If targets is not valid or if an engine is not registered.
248 If targets is not valid or if an engine is not registered.
249 """
249 """
250 if isinstance(targets, int):
250 if isinstance(targets, int):
251 if targets not in self.engines.keys():
251 if targets not in self.engines.keys():
252 log.msg("Engine with id %i is not registered" % targets)
252 log.msg("Engine with id %i is not registered" % targets)
253 raise error.InvalidEngineID("Engine with id %i is not registered" % targets)
253 raise error.InvalidEngineID("Engine with id %i is not registered" % targets)
254 else:
254 else:
255 return [self.engines[targets]]
255 return [self.engines[targets]]
256 elif isinstance(targets, (list, tuple)):
256 elif isinstance(targets, (list, tuple)):
257 for id in targets:
257 for id in targets:
258 if id not in self.engines.keys():
258 if id not in self.engines.keys():
259 log.msg("Engine with id %r is not registered" % id)
259 log.msg("Engine with id %r is not registered" % id)
260 raise error.InvalidEngineID("Engine with id %r is not registered" % id)
260 raise error.InvalidEngineID("Engine with id %r is not registered" % id)
261 return map(self.engines.get, targets)
261 return map(self.engines.get, targets)
262 elif targets == 'all':
262 elif targets == 'all':
263 eList = self.engines.values()
263 eList = self.engines.values()
264 if len(eList) == 0:
264 if len(eList) == 0:
265 msg = """There are no engines registered.
265 msg = """There are no engines registered.
266 Check the logs in ~/.ipython/log if you think there should have been."""
266 Check the logs in ~/.ipython/log if you think there should have been."""
267 raise error.NoEnginesRegistered(msg)
267 raise error.NoEnginesRegistered(msg)
268 else:
268 else:
269 return eList
269 return eList
270 else:
270 else:
271 raise error.InvalidEngineID("targets argument is not an int, list of ints or 'all': %r"%targets)
271 raise error.InvalidEngineID("targets argument is not an int, list of ints or 'all': %r"%targets)
272
272
273 def _performOnEngines(self, methodName, *args, **kwargs):
273 def _performOnEngines(self, methodName, *args, **kwargs):
274 """Calls a method on engines and returns deferred to list of results.
274 """Calls a method on engines and returns deferred to list of results.
275
275
276 :Parameters:
276 :Parameters:
277 methodName : str
277 methodName : str
278 Name of the method to be called.
278 Name of the method to be called.
279 targets : int, list of ints, 'all'
279 targets : int, list of ints, 'all'
280 The targets argument to be parsed into a list of engine objects.
280 The targets argument to be parsed into a list of engine objects.
281 args
281 args
282 The positional keyword arguments to be passed to the engines.
282 The positional keyword arguments to be passed to the engines.
283 kwargs
283 kwargs
284 The keyword arguments passed to the method
284 The keyword arguments passed to the method
285
285
286 :Returns: List of deferreds to the results on each engine
286 :Returns: List of deferreds to the results on each engine
287
287
288 :Exception:
288 :Exception:
289 InvalidEngineID
289 InvalidEngineID
290 If the targets argument is bad in any way.
290 If the targets argument is bad in any way.
291 AttributeError
291 AttributeError
292 If the method doesn't exist on one of the engines.
292 If the method doesn't exist on one of the engines.
293 """
293 """
294 targets = kwargs.pop('targets')
294 targets = kwargs.pop('targets')
295 log.msg("Performing %s on %r" % (methodName, targets))
295 log.msg("Performing %s on %r" % (methodName, targets))
296 # log.msg("Performing %s(%r, %r) on %r" % (methodName, args, kwargs, targets))
296 # log.msg("Performing %s(%r, %r) on %r" % (methodName, args, kwargs, targets))
297 # This will and should raise if targets is not valid!
297 # This will and should raise if targets is not valid!
298 engines = self.engineList(targets)
298 engines = self.engineList(targets)
299 dList = []
299 dList = []
300 for e in engines:
300 for e in engines:
301 meth = getattr(e, methodName, None)
301 meth = getattr(e, methodName, None)
302 if meth is not None:
302 if meth is not None:
303 dList.append(meth(*args, **kwargs))
303 dList.append(meth(*args, **kwargs))
304 else:
304 else:
305 raise AttributeError("Engine %i does not have method %s" % (e.id, methodName))
305 raise AttributeError("Engine %i does not have method %s" % (e.id, methodName))
306 return dList
306 return dList
307
307
308 def _performOnEnginesAndGatherBoth(self, methodName, *args, **kwargs):
308 def _performOnEnginesAndGatherBoth(self, methodName, *args, **kwargs):
309 """Called _performOnEngines and wraps result/exception into deferred."""
309 """Called _performOnEngines and wraps result/exception into deferred."""
310 try:
310 try:
311 dList = self._performOnEngines(methodName, *args, **kwargs)
311 dList = self._performOnEngines(methodName, *args, **kwargs)
312 except (error.InvalidEngineID, AttributeError, KeyError, error.NoEnginesRegistered):
312 except (error.InvalidEngineID, AttributeError, KeyError, error.NoEnginesRegistered):
313 return defer.fail(failure.Failure())
313 return defer.fail(failure.Failure())
314 else:
314 else:
315 # Having fireOnOneErrback is causing problems with the determinacy
315 # Having fireOnOneErrback is causing problems with the determinacy
316 # of the system. Basically, once a single engine has errbacked, this
316 # of the system. Basically, once a single engine has errbacked, this
317 # method returns. In some cases, this will cause client to submit
317 # method returns. In some cases, this will cause client to submit
318 # another command. Because the previous command is still running
318 # another command. Because the previous command is still running
319 # on some engines, this command will be queued. When those commands
319 # on some engines, this command will be queued. When those commands
320 # then errback, the second command will raise QueueCleared. Ahhh!
320 # then errback, the second command will raise QueueCleared. Ahhh!
321 d = gatherBoth(dList,
321 d = gatherBoth(dList,
322 fireOnOneErrback=0,
322 fireOnOneErrback=0,
323 consumeErrors=1,
323 consumeErrors=1,
324 logErrors=0)
324 logErrors=0)
325 d.addCallback(error.collect_exceptions, methodName)
325 d.addCallback(error.collect_exceptions, methodName)
326 return d
326 return d
327
327
328 #---------------------------------------------------------------------------
328 #---------------------------------------------------------------------------
329 # General IMultiEngine methods
329 # General IMultiEngine methods
330 #---------------------------------------------------------------------------
330 #---------------------------------------------------------------------------
331
331
332 def get_ids(self):
332 def get_ids(self):
333 return defer.succeed(self.engines.keys())
333 return defer.succeed(self.engines.keys())
334
334
335 #---------------------------------------------------------------------------
335 #---------------------------------------------------------------------------
336 # IEngineMultiplexer methods
336 # IEngineMultiplexer methods
337 #---------------------------------------------------------------------------
337 #---------------------------------------------------------------------------
338
338
339 def execute(self, lines, targets='all'):
339 def execute(self, lines, targets='all'):
340 return self._performOnEnginesAndGatherBoth('execute', lines, targets=targets)
340 return self._performOnEnginesAndGatherBoth('execute', lines, targets=targets)
341
341
342 def push(self, ns, targets='all'):
342 def push(self, ns, targets='all'):
343 return self._performOnEnginesAndGatherBoth('push', ns, targets=targets)
343 return self._performOnEnginesAndGatherBoth('push', ns, targets=targets)
344
344
345 def pull(self, keys, targets='all'):
345 def pull(self, keys, targets='all'):
346 return self._performOnEnginesAndGatherBoth('pull', keys, targets=targets)
346 return self._performOnEnginesAndGatherBoth('pull', keys, targets=targets)
347
347
348 def push_function(self, ns, targets='all'):
348 def push_function(self, ns, targets='all'):
349 return self._performOnEnginesAndGatherBoth('push_function', ns, targets=targets)
349 return self._performOnEnginesAndGatherBoth('push_function', ns, targets=targets)
350
350
351 def pull_function(self, keys, targets='all'):
351 def pull_function(self, keys, targets='all'):
352 return self._performOnEnginesAndGatherBoth('pull_function', keys, targets=targets)
352 return self._performOnEnginesAndGatherBoth('pull_function', keys, targets=targets)
353
353
354 def get_result(self, i=None, targets='all'):
354 def get_result(self, i=None, targets='all'):
355 return self._performOnEnginesAndGatherBoth('get_result', i, targets=targets)
355 return self._performOnEnginesAndGatherBoth('get_result', i, targets=targets)
356
356
357 def reset(self, targets='all'):
357 def reset(self, targets='all'):
358 return self._performOnEnginesAndGatherBoth('reset', targets=targets)
358 return self._performOnEnginesAndGatherBoth('reset', targets=targets)
359
359
360 def keys(self, targets='all'):
360 def keys(self, targets='all'):
361 return self._performOnEnginesAndGatherBoth('keys', targets=targets)
361 return self._performOnEnginesAndGatherBoth('keys', targets=targets)
362
362
363 def kill(self, controller=False, targets='all'):
363 def kill(self, controller=False, targets='all'):
364 if controller:
364 if controller:
365 targets = 'all'
365 targets = 'all'
366 d = self._performOnEnginesAndGatherBoth('kill', targets=targets)
366 d = self._performOnEnginesAndGatherBoth('kill', targets=targets)
367 if controller:
367 if controller:
368 log.msg("Killing controller")
368 log.msg("Killing controller")
369 d.addCallback(lambda _: reactor.callLater(2.0, reactor.stop))
369 d.addCallback(lambda _: reactor.callLater(2.0, reactor.stop))
370 # Consume any weird stuff coming back
370 # Consume any weird stuff coming back
371 d.addBoth(lambda _: None)
371 d.addBoth(lambda _: None)
372 return d
372 return d
373
373
374 def push_serialized(self, namespace, targets='all'):
374 def push_serialized(self, namespace, targets='all'):
375 for k, v in namespace.iteritems():
375 for k, v in namespace.iteritems():
376 log.msg("Pushed object %s is %f MB" % (k, v.getDataSize()))
376 log.msg("Pushed object %s is %f MB" % (k, v.getDataSize()))
377 d = self._performOnEnginesAndGatherBoth('push_serialized', namespace, targets=targets)
377 d = self._performOnEnginesAndGatherBoth('push_serialized', namespace, targets=targets)
378 return d
378 return d
379
379
380 def pull_serialized(self, keys, targets='all'):
380 def pull_serialized(self, keys, targets='all'):
381 try:
381 try:
382 dList = self._performOnEngines('pull_serialized', keys, targets=targets)
382 dList = self._performOnEngines('pull_serialized', keys, targets=targets)
383 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
383 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
384 return defer.fail(failure.Failure())
384 return defer.fail(failure.Failure())
385 else:
385 else:
386 for d in dList:
386 for d in dList:
387 d.addCallback(self._logSizes)
387 d.addCallback(self._logSizes)
388 d = gatherBoth(dList,
388 d = gatherBoth(dList,
389 fireOnOneErrback=0,
389 fireOnOneErrback=0,
390 consumeErrors=1,
390 consumeErrors=1,
391 logErrors=0)
391 logErrors=0)
392 d.addCallback(error.collect_exceptions, 'pull_serialized')
392 d.addCallback(error.collect_exceptions, 'pull_serialized')
393 return d
393 return d
394
394
395 def _logSizes(self, listOfSerialized):
395 def _logSizes(self, listOfSerialized):
396 if isinstance(listOfSerialized, (list, tuple)):
396 if isinstance(listOfSerialized, (list, tuple)):
397 for s in listOfSerialized:
397 for s in listOfSerialized:
398 log.msg("Pulled object is %f MB" % s.getDataSize())
398 log.msg("Pulled object is %f MB" % s.getDataSize())
399 else:
399 else:
400 log.msg("Pulled object is %f MB" % listOfSerialized.getDataSize())
400 log.msg("Pulled object is %f MB" % listOfSerialized.getDataSize())
401 return listOfSerialized
401 return listOfSerialized
402
402
403 def clear_queue(self, targets='all'):
403 def clear_queue(self, targets='all'):
404 return self._performOnEnginesAndGatherBoth('clear_queue', targets=targets)
404 return self._performOnEnginesAndGatherBoth('clear_queue', targets=targets)
405
405
406 def queue_status(self, targets='all'):
406 def queue_status(self, targets='all'):
407 log.msg("Getting queue status on %r" % targets)
407 log.msg("Getting queue status on %r" % targets)
408 try:
408 try:
409 engines = self.engineList(targets)
409 engines = self.engineList(targets)
410 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
410 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
411 return defer.fail(failure.Failure())
411 return defer.fail(failure.Failure())
412 else:
412 else:
413 dList = []
413 dList = []
414 for e in engines:
414 for e in engines:
415 dList.append(e.queue_status().addCallback(lambda s:(e.id, s)))
415 dList.append(e.queue_status().addCallback(lambda s:(e.id, s)))
416 d = gatherBoth(dList,
416 d = gatherBoth(dList,
417 fireOnOneErrback=0,
417 fireOnOneErrback=0,
418 consumeErrors=1,
418 consumeErrors=1,
419 logErrors=0)
419 logErrors=0)
420 d.addCallback(error.collect_exceptions, 'queue_status')
420 d.addCallback(error.collect_exceptions, 'queue_status')
421 return d
421 return d
422
422
423 def get_properties(self, keys=None, targets='all'):
423 def get_properties(self, keys=None, targets='all'):
424 log.msg("Getting properties on %r" % targets)
424 log.msg("Getting properties on %r" % targets)
425 try:
425 try:
426 engines = self.engineList(targets)
426 engines = self.engineList(targets)
427 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
427 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
428 return defer.fail(failure.Failure())
428 return defer.fail(failure.Failure())
429 else:
429 else:
430 dList = [e.get_properties(keys) for e in engines]
430 dList = [e.get_properties(keys) for e in engines]
431 d = gatherBoth(dList,
431 d = gatherBoth(dList,
432 fireOnOneErrback=0,
432 fireOnOneErrback=0,
433 consumeErrors=1,
433 consumeErrors=1,
434 logErrors=0)
434 logErrors=0)
435 d.addCallback(error.collect_exceptions, 'get_properties')
435 d.addCallback(error.collect_exceptions, 'get_properties')
436 return d
436 return d
437
437
438 def set_properties(self, properties, targets='all'):
438 def set_properties(self, properties, targets='all'):
439 log.msg("Setting properties on %r" % targets)
439 log.msg("Setting properties on %r" % targets)
440 try:
440 try:
441 engines = self.engineList(targets)
441 engines = self.engineList(targets)
442 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
442 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
443 return defer.fail(failure.Failure())
443 return defer.fail(failure.Failure())
444 else:
444 else:
445 dList = [e.set_properties(properties) for e in engines]
445 dList = [e.set_properties(properties) for e in engines]
446 d = gatherBoth(dList,
446 d = gatherBoth(dList,
447 fireOnOneErrback=0,
447 fireOnOneErrback=0,
448 consumeErrors=1,
448 consumeErrors=1,
449 logErrors=0)
449 logErrors=0)
450 d.addCallback(error.collect_exceptions, 'set_properties')
450 d.addCallback(error.collect_exceptions, 'set_properties')
451 return d
451 return d
452
452
453 def has_properties(self, keys, targets='all'):
453 def has_properties(self, keys, targets='all'):
454 log.msg("Checking properties on %r" % targets)
454 log.msg("Checking properties on %r" % targets)
455 try:
455 try:
456 engines = self.engineList(targets)
456 engines = self.engineList(targets)
457 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
457 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
458 return defer.fail(failure.Failure())
458 return defer.fail(failure.Failure())
459 else:
459 else:
460 dList = [e.has_properties(keys) for e in engines]
460 dList = [e.has_properties(keys) for e in engines]
461 d = gatherBoth(dList,
461 d = gatherBoth(dList,
462 fireOnOneErrback=0,
462 fireOnOneErrback=0,
463 consumeErrors=1,
463 consumeErrors=1,
464 logErrors=0)
464 logErrors=0)
465 d.addCallback(error.collect_exceptions, 'has_properties')
465 d.addCallback(error.collect_exceptions, 'has_properties')
466 return d
466 return d
467
467
468 def del_properties(self, keys, targets='all'):
468 def del_properties(self, keys, targets='all'):
469 log.msg("Deleting properties on %r" % targets)
469 log.msg("Deleting properties on %r" % targets)
470 try:
470 try:
471 engines = self.engineList(targets)
471 engines = self.engineList(targets)
472 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
472 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
473 return defer.fail(failure.Failure())
473 return defer.fail(failure.Failure())
474 else:
474 else:
475 dList = [e.del_properties(keys) for e in engines]
475 dList = [e.del_properties(keys) for e in engines]
476 d = gatherBoth(dList,
476 d = gatherBoth(dList,
477 fireOnOneErrback=0,
477 fireOnOneErrback=0,
478 consumeErrors=1,
478 consumeErrors=1,
479 logErrors=0)
479 logErrors=0)
480 d.addCallback(error.collect_exceptions, 'del_properties')
480 d.addCallback(error.collect_exceptions, 'del_properties')
481 return d
481 return d
482
482
483 def clear_properties(self, targets='all'):
483 def clear_properties(self, targets='all'):
484 log.msg("Clearing properties on %r" % targets)
484 log.msg("Clearing properties on %r" % targets)
485 try:
485 try:
486 engines = self.engineList(targets)
486 engines = self.engineList(targets)
487 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
487 except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
488 return defer.fail(failure.Failure())
488 return defer.fail(failure.Failure())
489 else:
489 else:
490 dList = [e.clear_properties() for e in engines]
490 dList = [e.clear_properties() for e in engines]
491 d = gatherBoth(dList,
491 d = gatherBoth(dList,
492 fireOnOneErrback=0,
492 fireOnOneErrback=0,
493 consumeErrors=1,
493 consumeErrors=1,
494 logErrors=0)
494 logErrors=0)
495 d.addCallback(error.collect_exceptions, 'clear_properties')
495 d.addCallback(error.collect_exceptions, 'clear_properties')
496 return d
496 return d
497
497
498
498
499 components.registerAdapter(MultiEngine,
499 components.registerAdapter(MultiEngine,
500 IControllerBase,
500 IControllerBase,
501 IMultiEngine)
501 IMultiEngine)
502
502
503
503
504 #-------------------------------------------------------------------------------
504 #-------------------------------------------------------------------------------
505 # Interfaces for the Synchronous MultiEngine
505 # Interfaces for the Synchronous MultiEngine
506 #-------------------------------------------------------------------------------
506 #-------------------------------------------------------------------------------
507
507
508 class ISynchronousEngineMultiplexer(Interface):
508 class ISynchronousEngineMultiplexer(Interface):
509 pass
509 pass
510
510
511
511
512 class ISynchronousMultiEngine(ISynchronousEngineMultiplexer):
512 class ISynchronousMultiEngine(ISynchronousEngineMultiplexer):
513 """Synchronous, two-phase version of IMultiEngine.
513 """Synchronous, two-phase version of IMultiEngine.
514
514
515 Methods in this interface are identical to those of IMultiEngine, but they
515 Methods in this interface are identical to those of IMultiEngine, but they
516 take one additional argument:
516 take one additional argument:
517
517
518 execute(lines, targets='all') -> execute(lines, targets='all, block=True)
518 execute(lines, targets='all') -> execute(lines, targets='all, block=True)
519
519
520 :Parameters:
520 :Parameters:
521 block : boolean
521 block : boolean
522 Should the method return a deferred to a deferredID or the
522 Should the method return a deferred to a deferredID or the
523 actual result. If block=False a deferred to a deferredID is
523 actual result. If block=False a deferred to a deferredID is
524 returned and the user must call `get_pending_deferred` at a later
524 returned and the user must call `get_pending_deferred` at a later
525 point. If block=True, a deferred to the actual result comes back.
525 point. If block=True, a deferred to the actual result comes back.
526 """
526 """
527 def get_pending_deferred(deferredID, block=True):
527 def get_pending_deferred(deferredID, block=True):
528 """"""
528 """"""
529
529
530 def clear_pending_deferreds():
530 def clear_pending_deferreds():
531 """"""
531 """"""
532
532
533
533
534 #-------------------------------------------------------------------------------
534 #-------------------------------------------------------------------------------
535 # Implementation of the Synchronous MultiEngine
535 # Implementation of the Synchronous MultiEngine
536 #-------------------------------------------------------------------------------
536 #-------------------------------------------------------------------------------
537
537
538 class SynchronousMultiEngine(PendingDeferredManager):
538 class SynchronousMultiEngine(PendingDeferredManager):
539 """Adapt an `IMultiEngine` -> `ISynchronousMultiEngine`
539 """Adapt an `IMultiEngine` -> `ISynchronousMultiEngine`
540
540
541 Warning, this class uses a decorator that currently uses **kwargs.
541 Warning, this class uses a decorator that currently uses **kwargs.
542 Because of this block must be passed as a kwarg, not positionally.
542 Because of this block must be passed as a kwarg, not positionally.
543 """
543 """
544
544
545 implements(ISynchronousMultiEngine)
545 implements(ISynchronousMultiEngine)
546
546
547 def __init__(self, multiengine):
547 def __init__(self, multiengine):
548 self.multiengine = multiengine
548 self.multiengine = multiengine
549 PendingDeferredManager.__init__(self)
549 PendingDeferredManager.__init__(self)
550
550
551 #---------------------------------------------------------------------------
551 #---------------------------------------------------------------------------
552 # Decorated pending deferred methods
552 # Decorated pending deferred methods
553 #---------------------------------------------------------------------------
553 #---------------------------------------------------------------------------
554
554
555 @two_phase
555 @two_phase
556 def execute(self, lines, targets='all'):
556 def execute(self, lines, targets='all'):
557 d = self.multiengine.execute(lines, targets)
557 d = self.multiengine.execute(lines, targets)
558 return d
558 return d
559
559
560 @two_phase
560 @two_phase
561 def push(self, namespace, targets='all'):
561 def push(self, namespace, targets='all'):
562 return self.multiengine.push(namespace, targets)
562 return self.multiengine.push(namespace, targets)
563
563
564 @two_phase
564 @two_phase
565 def pull(self, keys, targets='all'):
565 def pull(self, keys, targets='all'):
566 d = self.multiengine.pull(keys, targets)
566 d = self.multiengine.pull(keys, targets)
567 return d
567 return d
568
568
569 @two_phase
569 @two_phase
570 def push_function(self, namespace, targets='all'):
570 def push_function(self, namespace, targets='all'):
571 return self.multiengine.push_function(namespace, targets)
571 return self.multiengine.push_function(namespace, targets)
572
572
573 @two_phase
573 @two_phase
574 def pull_function(self, keys, targets='all'):
574 def pull_function(self, keys, targets='all'):
575 d = self.multiengine.pull_function(keys, targets)
575 d = self.multiengine.pull_function(keys, targets)
576 return d
576 return d
577
577
578 @two_phase
578 @two_phase
579 def get_result(self, i=None, targets='all'):
579 def get_result(self, i=None, targets='all'):
580 return self.multiengine.get_result(i, targets='all')
580 return self.multiengine.get_result(i, targets='all')
581
581
582 @two_phase
582 @two_phase
583 def reset(self, targets='all'):
583 def reset(self, targets='all'):
584 return self.multiengine.reset(targets)
584 return self.multiengine.reset(targets)
585
585
586 @two_phase
586 @two_phase
587 def keys(self, targets='all'):
587 def keys(self, targets='all'):
588 return self.multiengine.keys(targets)
588 return self.multiengine.keys(targets)
589
589
590 @two_phase
590 @two_phase
591 def kill(self, controller=False, targets='all'):
591 def kill(self, controller=False, targets='all'):
592 return self.multiengine.kill(controller, targets)
592 return self.multiengine.kill(controller, targets)
593
593
594 @two_phase
594 @two_phase
595 def push_serialized(self, namespace, targets='all'):
595 def push_serialized(self, namespace, targets='all'):
596 return self.multiengine.push_serialized(namespace, targets)
596 return self.multiengine.push_serialized(namespace, targets)
597
597
598 @two_phase
598 @two_phase
599 def pull_serialized(self, keys, targets='all'):
599 def pull_serialized(self, keys, targets='all'):
600 return self.multiengine.pull_serialized(keys, targets)
600 return self.multiengine.pull_serialized(keys, targets)
601
601
602 @two_phase
602 @two_phase
603 def clear_queue(self, targets='all'):
603 def clear_queue(self, targets='all'):
604 return self.multiengine.clear_queue(targets)
604 return self.multiengine.clear_queue(targets)
605
605
606 @two_phase
606 @two_phase
607 def queue_status(self, targets='all'):
607 def queue_status(self, targets='all'):
608 return self.multiengine.queue_status(targets)
608 return self.multiengine.queue_status(targets)
609
609
610 @two_phase
610 @two_phase
611 def set_properties(self, properties, targets='all'):
611 def set_properties(self, properties, targets='all'):
612 return self.multiengine.set_properties(properties, targets)
612 return self.multiengine.set_properties(properties, targets)
613
613
614 @two_phase
614 @two_phase
615 def get_properties(self, keys=None, targets='all'):
615 def get_properties(self, keys=None, targets='all'):
616 return self.multiengine.get_properties(keys, targets)
616 return self.multiengine.get_properties(keys, targets)
617
617
618 @two_phase
618 @two_phase
619 def has_properties(self, keys, targets='all'):
619 def has_properties(self, keys, targets='all'):
620 return self.multiengine.has_properties(keys, targets)
620 return self.multiengine.has_properties(keys, targets)
621
621
622 @two_phase
622 @two_phase
623 def del_properties(self, keys, targets='all'):
623 def del_properties(self, keys, targets='all'):
624 return self.multiengine.del_properties(keys, targets)
624 return self.multiengine.del_properties(keys, targets)
625
625
626 @two_phase
626 @two_phase
627 def clear_properties(self, targets='all'):
627 def clear_properties(self, targets='all'):
628 return self.multiengine.clear_properties(targets)
628 return self.multiengine.clear_properties(targets)
629
629
630 #---------------------------------------------------------------------------
630 #---------------------------------------------------------------------------
631 # IMultiEngine methods
631 # IMultiEngine methods
632 #---------------------------------------------------------------------------
632 #---------------------------------------------------------------------------
633
633
634 def get_ids(self):
634 def get_ids(self):
635 """Return a list of registered engine ids.
635 """Return a list of registered engine ids.
636
636
637 Never use the two phase block/non-block stuff for this.
637 Never use the two phase block/non-block stuff for this.
638 """
638 """
639 return self.multiengine.get_ids()
639 return self.multiengine.get_ids()
640
640
641
641
642 components.registerAdapter(SynchronousMultiEngine, IMultiEngine, ISynchronousMultiEngine)
642 components.registerAdapter(SynchronousMultiEngine, IMultiEngine, ISynchronousMultiEngine)
643
643
644
644
645 #-------------------------------------------------------------------------------
645 #-------------------------------------------------------------------------------
646 # Various high-level interfaces that can be used as MultiEngine mix-ins
646 # Various high-level interfaces that can be used as MultiEngine mix-ins
647 #-------------------------------------------------------------------------------
647 #-------------------------------------------------------------------------------
648
648
649 #-------------------------------------------------------------------------------
649 #-------------------------------------------------------------------------------
650 # IMultiEngineCoordinator
650 # IMultiEngineCoordinator
651 #-------------------------------------------------------------------------------
651 #-------------------------------------------------------------------------------
652
652
653 class IMultiEngineCoordinator(Interface):
653 class IMultiEngineCoordinator(Interface):
654 """Methods that work on multiple engines explicitly."""
654 """Methods that work on multiple engines explicitly."""
655
655
656 def scatter(key, seq, dist='b', flatten=False, targets='all'):
656 def scatter(key, seq, dist='b', flatten=False, targets='all'):
657 """Partition and distribute a sequence to targets."""
657 """Partition and distribute a sequence to targets."""
658
658
659 def gather(key, dist='b', targets='all'):
659 def gather(key, dist='b', targets='all'):
660 """Gather object key from targets."""
660 """Gather object key from targets."""
661
661
662 def raw_map(func, seqs, dist='b', targets='all'):
662 def raw_map(func, seqs, dist='b', targets='all'):
663 """
663 """
664 A parallelized version of Python's builtin `map` function.
664 A parallelized version of Python's builtin `map` function.
665
665
666 This has a slightly different syntax than the builtin `map`.
666 This has a slightly different syntax than the builtin `map`.
667 This is needed because we need to have keyword arguments and thus
667 This is needed because we need to have keyword arguments and thus
668 can't use *args to capture all the sequences. Instead, they must
668 can't use *args to capture all the sequences. Instead, they must
669 be passed in a list or tuple.
669 be passed in a list or tuple.
670
670
671 The equivalence is:
671 The equivalence is:
672
672
673 raw_map(func, seqs) -> map(func, seqs[0], seqs[1], ...)
673 raw_map(func, seqs) -> map(func, seqs[0], seqs[1], ...)
674
674
675 Most users will want to use parallel functions or the `mapper`
675 Most users will want to use parallel functions or the `mapper`
676 and `map` methods for an API that follows that of the builtin
676 and `map` methods for an API that follows that of the builtin
677 `map`.
677 `map`.
678 """
678 """
679
679
680
680
681 class ISynchronousMultiEngineCoordinator(IMultiEngineCoordinator):
681 class ISynchronousMultiEngineCoordinator(IMultiEngineCoordinator):
682 """Methods that work on multiple engines explicitly."""
682 """Methods that work on multiple engines explicitly."""
683
683
684 def scatter(key, seq, dist='b', flatten=False, targets='all', block=True):
684 def scatter(key, seq, dist='b', flatten=False, targets='all', block=True):
685 """Partition and distribute a sequence to targets."""
685 """Partition and distribute a sequence to targets."""
686
686
687 def gather(key, dist='b', targets='all', block=True):
687 def gather(key, dist='b', targets='all', block=True):
688 """Gather object key from targets"""
688 """Gather object key from targets"""
689
689
690 def raw_map(func, seqs, dist='b', targets='all', block=True):
690 def raw_map(func, seqs, dist='b', targets='all', block=True):
691 """
691 """
692 A parallelized version of Python's builtin map.
692 A parallelized version of Python's builtin map.
693
693
694 This has a slightly different syntax than the builtin `map`.
694 This has a slightly different syntax than the builtin `map`.
695 This is needed because we need to have keyword arguments and thus
695 This is needed because we need to have keyword arguments and thus
696 can't use *args to capture all the sequences. Instead, they must
696 can't use *args to capture all the sequences. Instead, they must
697 be passed in a list or tuple.
697 be passed in a list or tuple.
698
698
699 raw_map(func, seqs) -> map(func, seqs[0], seqs[1], ...)
699 raw_map(func, seqs) -> map(func, seqs[0], seqs[1], ...)
700
700
701 Most users will want to use parallel functions or the `mapper`
701 Most users will want to use parallel functions or the `mapper`
702 and `map` methods for an API that follows that of the builtin
702 and `map` methods for an API that follows that of the builtin
703 `map`.
703 `map`.
704 """
704 """
705
705
706
706
707 #-------------------------------------------------------------------------------
707 #-------------------------------------------------------------------------------
708 # IMultiEngineExtras
708 # IMultiEngineExtras
709 #-------------------------------------------------------------------------------
709 #-------------------------------------------------------------------------------
710
710
711 class IMultiEngineExtras(Interface):
711 class IMultiEngineExtras(Interface):
712
712
713 def zip_pull(targets, keys):
713 def zip_pull(targets, keys):
714 """
714 """
715 Pull, but return results in a different format from `pull`.
715 Pull, but return results in a different format from `pull`.
716
716
717 This method basically returns zip(pull(targets, *keys)), with a few
717 This method basically returns zip(pull(targets, *keys)), with a few
718 edge cases handled differently. Users of chainsaw will find this format
718 edge cases handled differently. Users of chainsaw will find this format
719 familiar.
719 familiar.
720 """
720 """
721
721
722 def run(targets, fname):
722 def run(targets, fname):
723 """Run a .py file on targets."""
723 """Run a .py file on targets."""
724
724
725
725
726 class ISynchronousMultiEngineExtras(IMultiEngineExtras):
726 class ISynchronousMultiEngineExtras(IMultiEngineExtras):
727 def zip_pull(targets, keys, block=True):
727 def zip_pull(targets, keys, block=True):
728 """
728 """
729 Pull, but return results in a different format from `pull`.
729 Pull, but return results in a different format from `pull`.
730
730
731 This method basically returns zip(pull(targets, *keys)), with a few
731 This method basically returns zip(pull(targets, *keys)), with a few
732 edge cases handled differently. Users of chainsaw will find this format
732 edge cases handled differently. Users of chainsaw will find this format
733 familiar.
733 familiar.
734 """
734 """
735
735
736 def run(targets, fname, block=True):
736 def run(targets, fname, block=True):
737 """Run a .py file on targets."""
737 """Run a .py file on targets."""
738
738
739 #-------------------------------------------------------------------------------
739 #-------------------------------------------------------------------------------
740 # The full MultiEngine interface
740 # The full MultiEngine interface
741 #-------------------------------------------------------------------------------
741 #-------------------------------------------------------------------------------
742
742
743 class IFullMultiEngine(IMultiEngine,
743 class IFullMultiEngine(IMultiEngine,
744 IMultiEngineCoordinator,
744 IMultiEngineCoordinator,
745 IMultiEngineExtras):
745 IMultiEngineExtras):
746 pass
746 pass
747
747
748
748
749 class IFullSynchronousMultiEngine(ISynchronousMultiEngine,
749 class IFullSynchronousMultiEngine(ISynchronousMultiEngine,
750 ISynchronousMultiEngineCoordinator,
750 ISynchronousMultiEngineCoordinator,
751 ISynchronousMultiEngineExtras):
751 ISynchronousMultiEngineExtras):
752 pass
752 pass
753
753
@@ -1,178 +1,178 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 # -*- test-case-name: IPython.kernel.test.test_pendingdeferred -*-
2 # -*- test-case-name: IPython.kernel.test.test_pendingdeferred -*-
3
3
4 """Classes to manage pending Deferreds.
4 """Classes to manage pending Deferreds.
5
5
6 A pending deferred is a deferred that may or may not have fired. This module
6 A pending deferred is a deferred that may or may not have fired. This module
7 is useful for taking a class whose methods return deferreds and wrapping it to
7 is useful for taking a class whose methods return deferreds and wrapping it to
8 provide API that keeps track of those deferreds for later retrieval. See the
8 provide API that keeps track of those deferreds for later retrieval. See the
9 tests for examples of its usage.
9 tests for examples of its usage.
10 """
10 """
11
11
12 __docformat__ = "restructuredtext en"
12 __docformat__ = "restructuredtext en"
13
13
14 #-------------------------------------------------------------------------------
14 #-------------------------------------------------------------------------------
15 # Copyright (C) 2008 The IPython Development Team
15 # Copyright (C) 2008 The IPython Development Team
16 #
16 #
17 # Distributed under the terms of the BSD License. The full license is in
17 # Distributed under the terms of the BSD License. The full license is in
18 # the file COPYING, distributed as part of this software.
18 # the file COPYING, distributed as part of this software.
19 #-------------------------------------------------------------------------------
19 #-------------------------------------------------------------------------------
20
20
21 #-------------------------------------------------------------------------------
21 #-------------------------------------------------------------------------------
22 # Imports
22 # Imports
23 #-------------------------------------------------------------------------------
23 #-------------------------------------------------------------------------------
24
24
25 from twisted.application import service
25 from twisted.application import service
26 from twisted.internet import defer, reactor
26 from twisted.internet import defer, reactor
27 from twisted.python import log, components, failure
27 from twisted.python import log, components, failure
28 from zope.interface import Interface, implements, Attribute
28 from zope.interface import Interface, implements, Attribute
29
29
30 from IPython.kernel.twistedutil import gatherBoth
30 from IPython.kernel.twistedutil import gatherBoth
31 from IPython.kernel import error
31 from IPython.kernel import error
32 from IPython.external import guid
32 from IPython.external import guid
33 from IPython.tools import growl
33 from IPython.utils import growl
34
34
35 class PendingDeferredManager(object):
35 class PendingDeferredManager(object):
36 """A class to track pending deferreds.
36 """A class to track pending deferreds.
37
37
38 To track a pending deferred, the user of this class must first
38 To track a pending deferred, the user of this class must first
39 get a deferredID by calling `get_next_deferred_id`. Then the user
39 get a deferredID by calling `get_next_deferred_id`. Then the user
40 calls `save_pending_deferred` passing that id and the deferred to
40 calls `save_pending_deferred` passing that id and the deferred to
41 be tracked. To later retrieve it, the user calls
41 be tracked. To later retrieve it, the user calls
42 `get_pending_deferred` passing the id.
42 `get_pending_deferred` passing the id.
43 """
43 """
44
44
45 def __init__(self):
45 def __init__(self):
46 """Manage pending deferreds."""
46 """Manage pending deferreds."""
47
47
48 self.results = {} # Populated when results are ready
48 self.results = {} # Populated when results are ready
49 self.deferred_ids = [] # List of deferred ids I am managing
49 self.deferred_ids = [] # List of deferred ids I am managing
50 self.deferreds_to_callback = {} # dict of lists of deferreds to callback
50 self.deferreds_to_callback = {} # dict of lists of deferreds to callback
51
51
52 def get_deferred_id(self):
52 def get_deferred_id(self):
53 return guid.generate()
53 return guid.generate()
54
54
55 def quick_has_id(self, deferred_id):
55 def quick_has_id(self, deferred_id):
56 return deferred_id in self.deferred_ids
56 return deferred_id in self.deferred_ids
57
57
58 def _save_result(self, result, deferred_id):
58 def _save_result(self, result, deferred_id):
59 if self.quick_has_id(deferred_id):
59 if self.quick_has_id(deferred_id):
60 self.results[deferred_id] = result
60 self.results[deferred_id] = result
61 self._trigger_callbacks(deferred_id)
61 self._trigger_callbacks(deferred_id)
62
62
63 def _trigger_callbacks(self, deferred_id):
63 def _trigger_callbacks(self, deferred_id):
64 # Go through and call the waiting callbacks
64 # Go through and call the waiting callbacks
65 result = self.results.get(deferred_id)
65 result = self.results.get(deferred_id)
66 if result is not None: # Only trigger if there is a result
66 if result is not None: # Only trigger if there is a result
67 try:
67 try:
68 d = self.deferreds_to_callback.pop(deferred_id)
68 d = self.deferreds_to_callback.pop(deferred_id)
69 except KeyError:
69 except KeyError:
70 d = None
70 d = None
71 if d is not None:
71 if d is not None:
72 if isinstance(result, failure.Failure):
72 if isinstance(result, failure.Failure):
73 d.errback(result)
73 d.errback(result)
74 else:
74 else:
75 d.callback(result)
75 d.callback(result)
76 self.delete_pending_deferred(deferred_id)
76 self.delete_pending_deferred(deferred_id)
77
77
78 def save_pending_deferred(self, d, deferred_id=None):
78 def save_pending_deferred(self, d, deferred_id=None):
79 """Save the result of a deferred for later retrieval.
79 """Save the result of a deferred for later retrieval.
80
80
81 This works even if the deferred has not fired.
81 This works even if the deferred has not fired.
82
82
83 Only callbacks and errbacks applied to d before this method
83 Only callbacks and errbacks applied to d before this method
84 is called will be called no the final result.
84 is called will be called no the final result.
85 """
85 """
86 if deferred_id is None:
86 if deferred_id is None:
87 deferred_id = self.get_deferred_id()
87 deferred_id = self.get_deferred_id()
88 self.deferred_ids.append(deferred_id)
88 self.deferred_ids.append(deferred_id)
89 d.addBoth(self._save_result, deferred_id)
89 d.addBoth(self._save_result, deferred_id)
90 return deferred_id
90 return deferred_id
91
91
92 def _protected_del(self, key, container):
92 def _protected_del(self, key, container):
93 try:
93 try:
94 del container[key]
94 del container[key]
95 except Exception:
95 except Exception:
96 pass
96 pass
97
97
98 def delete_pending_deferred(self, deferred_id):
98 def delete_pending_deferred(self, deferred_id):
99 """Remove a deferred I am tracking and add a null Errback.
99 """Remove a deferred I am tracking and add a null Errback.
100
100
101 :Parameters:
101 :Parameters:
102 deferredID : str
102 deferredID : str
103 The id of a deferred that I am tracking.
103 The id of a deferred that I am tracking.
104 """
104 """
105 if self.quick_has_id(deferred_id):
105 if self.quick_has_id(deferred_id):
106 # First go through a errback any deferreds that are still waiting
106 # First go through a errback any deferreds that are still waiting
107 d = self.deferreds_to_callback.get(deferred_id)
107 d = self.deferreds_to_callback.get(deferred_id)
108 if d is not None:
108 if d is not None:
109 d.errback(failure.Failure(error.AbortedPendingDeferredError("pending deferred has been deleted: %r"%deferred_id)))
109 d.errback(failure.Failure(error.AbortedPendingDeferredError("pending deferred has been deleted: %r"%deferred_id)))
110 # Now delete all references to this deferred_id
110 # Now delete all references to this deferred_id
111 ind = self.deferred_ids.index(deferred_id)
111 ind = self.deferred_ids.index(deferred_id)
112 self._protected_del(ind, self.deferred_ids)
112 self._protected_del(ind, self.deferred_ids)
113 self._protected_del(deferred_id, self.deferreds_to_callback)
113 self._protected_del(deferred_id, self.deferreds_to_callback)
114 self._protected_del(deferred_id, self.results)
114 self._protected_del(deferred_id, self.results)
115 else:
115 else:
116 raise error.InvalidDeferredID('invalid deferred_id: %r' % deferred_id)
116 raise error.InvalidDeferredID('invalid deferred_id: %r' % deferred_id)
117
117
118 def clear_pending_deferreds(self):
118 def clear_pending_deferreds(self):
119 """Remove all the deferreds I am tracking."""
119 """Remove all the deferreds I am tracking."""
120 for did in self.deferred_ids:
120 for did in self.deferred_ids:
121 self.delete_pending_deferred(did)
121 self.delete_pending_deferred(did)
122
122
123 def _delete_and_pass_through(self, r, deferred_id):
123 def _delete_and_pass_through(self, r, deferred_id):
124 self.delete_pending_deferred(deferred_id)
124 self.delete_pending_deferred(deferred_id)
125 return r
125 return r
126
126
127 def get_pending_deferred(self, deferred_id, block):
127 def get_pending_deferred(self, deferred_id, block):
128 if not self.quick_has_id(deferred_id) or self.deferreds_to_callback.get(deferred_id) is not None:
128 if not self.quick_has_id(deferred_id) or self.deferreds_to_callback.get(deferred_id) is not None:
129 return defer.fail(failure.Failure(error.InvalidDeferredID('invalid deferred_id: %r' + deferred_id)))
129 return defer.fail(failure.Failure(error.InvalidDeferredID('invalid deferred_id: %r' + deferred_id)))
130 result = self.results.get(deferred_id)
130 result = self.results.get(deferred_id)
131 if result is not None:
131 if result is not None:
132 self.delete_pending_deferred(deferred_id)
132 self.delete_pending_deferred(deferred_id)
133 if isinstance(result, failure.Failure):
133 if isinstance(result, failure.Failure):
134 return defer.fail(result)
134 return defer.fail(result)
135 else:
135 else:
136 return defer.succeed(result)
136 return defer.succeed(result)
137 else: # Result is not ready
137 else: # Result is not ready
138 if block:
138 if block:
139 d = defer.Deferred()
139 d = defer.Deferred()
140 self.deferreds_to_callback[deferred_id] = d
140 self.deferreds_to_callback[deferred_id] = d
141 return d
141 return d
142 else:
142 else:
143 return defer.fail(failure.Failure(error.ResultNotCompleted("result not completed: %r" % deferred_id)))
143 return defer.fail(failure.Failure(error.ResultNotCompleted("result not completed: %r" % deferred_id)))
144
144
145 def two_phase(wrapped_method):
145 def two_phase(wrapped_method):
146 """Wrap methods that return a deferred into a two phase process.
146 """Wrap methods that return a deferred into a two phase process.
147
147
148 This transforms::
148 This transforms::
149
149
150 foo(arg1, arg2, ...) -> foo(arg1, arg2,...,block=True).
150 foo(arg1, arg2, ...) -> foo(arg1, arg2,...,block=True).
151
151
152 The wrapped method will then return a deferred to a deferred id. This will
152 The wrapped method will then return a deferred to a deferred id. This will
153 only work on method of classes that inherit from `PendingDeferredManager`,
153 only work on method of classes that inherit from `PendingDeferredManager`,
154 as that class provides an API for
154 as that class provides an API for
155
155
156 block is a boolean to determine if we should use the two phase process or
156 block is a boolean to determine if we should use the two phase process or
157 just simply call the wrapped method. At this point block does not have a
157 just simply call the wrapped method. At this point block does not have a
158 default and it probably won't.
158 default and it probably won't.
159 """
159 """
160
160
161 def wrapper_two_phase(pdm, *args, **kwargs):
161 def wrapper_two_phase(pdm, *args, **kwargs):
162 try:
162 try:
163 block = kwargs.pop('block')
163 block = kwargs.pop('block')
164 except KeyError:
164 except KeyError:
165 block = True # The default if not specified
165 block = True # The default if not specified
166 if block:
166 if block:
167 return wrapped_method(pdm, *args, **kwargs)
167 return wrapped_method(pdm, *args, **kwargs)
168 else:
168 else:
169 d = wrapped_method(pdm, *args, **kwargs)
169 d = wrapped_method(pdm, *args, **kwargs)
170 deferred_id=pdm.save_pending_deferred(d)
170 deferred_id=pdm.save_pending_deferred(d)
171 return defer.succeed(deferred_id)
171 return defer.succeed(deferred_id)
172
172
173 return wrapper_two_phase
173 return wrapper_two_phase
174
174
175
175
176
176
177
177
178
178
@@ -1,416 +1,416 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3
3
4 """The IPython controller."""
4 """The IPython controller."""
5
5
6 __docformat__ = "restructuredtext en"
6 __docformat__ = "restructuredtext en"
7
7
8 #-------------------------------------------------------------------------------
8 #-------------------------------------------------------------------------------
9 # Copyright (C) 2008 The IPython Development Team
9 # Copyright (C) 2008 The IPython Development Team
10 #
10 #
11 # Distributed under the terms of the BSD License. The full license is in
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
12 # the file COPYING, distributed as part of this software.
13 #-------------------------------------------------------------------------------
13 #-------------------------------------------------------------------------------
14
14
15 #-------------------------------------------------------------------------------
15 #-------------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-------------------------------------------------------------------------------
17 #-------------------------------------------------------------------------------
18
18
19 # Python looks for an empty string at the beginning of sys.path to enable
19 # Python looks for an empty string at the beginning of sys.path to enable
20 # importing from the cwd.
20 # importing from the cwd.
21 import sys
21 import sys
22 sys.path.insert(0, '')
22 sys.path.insert(0, '')
23
23
24 from optparse import OptionParser
24 from optparse import OptionParser
25 import os
25 import os
26 import time
26 import time
27 import tempfile
27 import tempfile
28
28
29 from twisted.application import internet, service
29 from twisted.application import internet, service
30 from twisted.internet import reactor, error, defer
30 from twisted.internet import reactor, error, defer
31 from twisted.python import log
31 from twisted.python import log
32
32
33 from IPython.kernel.fcutil import Tub, UnauthenticatedTub, have_crypto
33 from IPython.kernel.fcutil import Tub, UnauthenticatedTub, have_crypto
34
34
35 # from IPython.tools import growl
35 # from IPython.utils import growl
36 # growl.start("IPython1 Controller")
36 # growl.start("IPython1 Controller")
37
37
38 from IPython.kernel.error import SecurityError
38 from IPython.kernel.error import SecurityError
39 from IPython.kernel import controllerservice
39 from IPython.kernel import controllerservice
40 from IPython.kernel.fcutil import check_furl_file_security
40 from IPython.kernel.fcutil import check_furl_file_security
41
41
42 # Create various ipython directories if they don't exist.
42 # Create various ipython directories if they don't exist.
43 # This must be done before IPython.kernel.config is imported.
43 # This must be done before IPython.kernel.config is imported.
44 from IPython.core.iplib import user_setup
44 from IPython.core.iplib import user_setup
45 from IPython.utils.genutils import get_ipython_dir, get_log_dir, get_security_dir
45 from IPython.utils.genutils import get_ipython_dir, get_log_dir, get_security_dir
46 if os.name == 'posix':
46 if os.name == 'posix':
47 rc_suffix = ''
47 rc_suffix = ''
48 else:
48 else:
49 rc_suffix = '.ini'
49 rc_suffix = '.ini'
50 user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
50 user_setup(get_ipython_dir(), rc_suffix, mode='install', interactive=False)
51 get_log_dir()
51 get_log_dir()
52 get_security_dir()
52 get_security_dir()
53
53
54 from IPython.kernel.config import config_manager as kernel_config_manager
54 from IPython.kernel.config import config_manager as kernel_config_manager
55 from IPython.config.cutils import import_item
55 from IPython.config.cutils import import_item
56
56
57
57
58 #-------------------------------------------------------------------------------
58 #-------------------------------------------------------------------------------
59 # Code
59 # Code
60 #-------------------------------------------------------------------------------
60 #-------------------------------------------------------------------------------
61
61
62 def get_temp_furlfile(filename):
62 def get_temp_furlfile(filename):
63 return tempfile.mktemp(dir=os.path.dirname(filename),
63 return tempfile.mktemp(dir=os.path.dirname(filename),
64 prefix=os.path.basename(filename))
64 prefix=os.path.basename(filename))
65
65
66 def make_tub(ip, port, secure, cert_file):
66 def make_tub(ip, port, secure, cert_file):
67 """
67 """
68 Create a listening tub given an ip, port, and cert_file location.
68 Create a listening tub given an ip, port, and cert_file location.
69
69
70 :Parameters:
70 :Parameters:
71 ip : str
71 ip : str
72 The ip address that the tub should listen on. Empty means all
72 The ip address that the tub should listen on. Empty means all
73 port : int
73 port : int
74 The port that the tub should listen on. A value of 0 means
74 The port that the tub should listen on. A value of 0 means
75 pick a random port
75 pick a random port
76 secure: boolean
76 secure: boolean
77 Will the connection be secure (in the foolscap sense)
77 Will the connection be secure (in the foolscap sense)
78 cert_file:
78 cert_file:
79 A filename of a file to be used for theSSL certificate
79 A filename of a file to be used for theSSL certificate
80 """
80 """
81 if secure:
81 if secure:
82 if have_crypto:
82 if have_crypto:
83 tub = Tub(certFile=cert_file)
83 tub = Tub(certFile=cert_file)
84 else:
84 else:
85 raise SecurityError("""
85 raise SecurityError("""
86 OpenSSL/pyOpenSSL is not available, so we can't run in secure mode.
86 OpenSSL/pyOpenSSL is not available, so we can't run in secure mode.
87 Try running without security using 'ipcontroller -xy'.
87 Try running without security using 'ipcontroller -xy'.
88 """)
88 """)
89 else:
89 else:
90 tub = UnauthenticatedTub()
90 tub = UnauthenticatedTub()
91
91
92 # Set the strport based on the ip and port and start listening
92 # Set the strport based on the ip and port and start listening
93 if ip == '':
93 if ip == '':
94 strport = "tcp:%i" % port
94 strport = "tcp:%i" % port
95 else:
95 else:
96 strport = "tcp:%i:interface=%s" % (port, ip)
96 strport = "tcp:%i:interface=%s" % (port, ip)
97 listener = tub.listenOn(strport)
97 listener = tub.listenOn(strport)
98
98
99 return tub, listener
99 return tub, listener
100
100
101 def make_client_service(controller_service, config):
101 def make_client_service(controller_service, config):
102 """
102 """
103 Create a service that will listen for clients.
103 Create a service that will listen for clients.
104
104
105 This service is simply a `foolscap.Tub` instance that has a set of Referenceables
105 This service is simply a `foolscap.Tub` instance that has a set of Referenceables
106 registered with it.
106 registered with it.
107 """
107 """
108
108
109 # Now create the foolscap tub
109 # Now create the foolscap tub
110 ip = config['controller']['client_tub']['ip']
110 ip = config['controller']['client_tub']['ip']
111 port = config['controller']['client_tub'].as_int('port')
111 port = config['controller']['client_tub'].as_int('port')
112 location = config['controller']['client_tub']['location']
112 location = config['controller']['client_tub']['location']
113 secure = config['controller']['client_tub']['secure']
113 secure = config['controller']['client_tub']['secure']
114 cert_file = config['controller']['client_tub']['cert_file']
114 cert_file = config['controller']['client_tub']['cert_file']
115 client_tub, client_listener = make_tub(ip, port, secure, cert_file)
115 client_tub, client_listener = make_tub(ip, port, secure, cert_file)
116
116
117 # Set the location in the trivial case of localhost
117 # Set the location in the trivial case of localhost
118 if ip == 'localhost' or ip == '127.0.0.1':
118 if ip == 'localhost' or ip == '127.0.0.1':
119 location = "127.0.0.1"
119 location = "127.0.0.1"
120
120
121 if not secure:
121 if not secure:
122 log.msg("WARNING: you are running the controller with no client security")
122 log.msg("WARNING: you are running the controller with no client security")
123
123
124 def set_location_and_register():
124 def set_location_and_register():
125 """Set the location for the tub and return a deferred."""
125 """Set the location for the tub and return a deferred."""
126
126
127 def register(empty, ref, furl_file):
127 def register(empty, ref, furl_file):
128 # We create and then move to make sure that when the file
128 # We create and then move to make sure that when the file
129 # appears to other processes, the buffer has the flushed
129 # appears to other processes, the buffer has the flushed
130 # and the file has been closed
130 # and the file has been closed
131 temp_furl_file = get_temp_furlfile(furl_file)
131 temp_furl_file = get_temp_furlfile(furl_file)
132 client_tub.registerReference(ref, furlFile=temp_furl_file)
132 client_tub.registerReference(ref, furlFile=temp_furl_file)
133 os.rename(temp_furl_file, furl_file)
133 os.rename(temp_furl_file, furl_file)
134
134
135 if location == '':
135 if location == '':
136 d = client_tub.setLocationAutomatically()
136 d = client_tub.setLocationAutomatically()
137 else:
137 else:
138 d = defer.maybeDeferred(client_tub.setLocation, "%s:%i" % (location, client_listener.getPortnum()))
138 d = defer.maybeDeferred(client_tub.setLocation, "%s:%i" % (location, client_listener.getPortnum()))
139
139
140 for ciname, ci in config['controller']['controller_interfaces'].iteritems():
140 for ciname, ci in config['controller']['controller_interfaces'].iteritems():
141 log.msg("Adapting Controller to interface: %s" % ciname)
141 log.msg("Adapting Controller to interface: %s" % ciname)
142 furl_file = ci['furl_file']
142 furl_file = ci['furl_file']
143 log.msg("Saving furl for interface [%s] to file: %s" % (ciname, furl_file))
143 log.msg("Saving furl for interface [%s] to file: %s" % (ciname, furl_file))
144 check_furl_file_security(furl_file, secure)
144 check_furl_file_security(furl_file, secure)
145 adapted_controller = import_item(ci['controller_interface'])(controller_service)
145 adapted_controller = import_item(ci['controller_interface'])(controller_service)
146 d.addCallback(register, import_item(ci['fc_interface'])(adapted_controller),
146 d.addCallback(register, import_item(ci['fc_interface'])(adapted_controller),
147 furl_file=ci['furl_file'])
147 furl_file=ci['furl_file'])
148
148
149 reactor.callWhenRunning(set_location_and_register)
149 reactor.callWhenRunning(set_location_and_register)
150 return client_tub
150 return client_tub
151
151
152
152
153 def make_engine_service(controller_service, config):
153 def make_engine_service(controller_service, config):
154 """
154 """
155 Create a service that will listen for engines.
155 Create a service that will listen for engines.
156
156
157 This service is simply a `foolscap.Tub` instance that has a set of Referenceables
157 This service is simply a `foolscap.Tub` instance that has a set of Referenceables
158 registered with it.
158 registered with it.
159 """
159 """
160
160
161 # Now create the foolscap tub
161 # Now create the foolscap tub
162 ip = config['controller']['engine_tub']['ip']
162 ip = config['controller']['engine_tub']['ip']
163 port = config['controller']['engine_tub'].as_int('port')
163 port = config['controller']['engine_tub'].as_int('port')
164 location = config['controller']['engine_tub']['location']
164 location = config['controller']['engine_tub']['location']
165 secure = config['controller']['engine_tub']['secure']
165 secure = config['controller']['engine_tub']['secure']
166 cert_file = config['controller']['engine_tub']['cert_file']
166 cert_file = config['controller']['engine_tub']['cert_file']
167 engine_tub, engine_listener = make_tub(ip, port, secure, cert_file)
167 engine_tub, engine_listener = make_tub(ip, port, secure, cert_file)
168
168
169 # Set the location in the trivial case of localhost
169 # Set the location in the trivial case of localhost
170 if ip == 'localhost' or ip == '127.0.0.1':
170 if ip == 'localhost' or ip == '127.0.0.1':
171 location = "127.0.0.1"
171 location = "127.0.0.1"
172
172
173 if not secure:
173 if not secure:
174 log.msg("WARNING: you are running the controller with no engine security")
174 log.msg("WARNING: you are running the controller with no engine security")
175
175
176 def set_location_and_register():
176 def set_location_and_register():
177 """Set the location for the tub and return a deferred."""
177 """Set the location for the tub and return a deferred."""
178
178
179 def register(empty, ref, furl_file):
179 def register(empty, ref, furl_file):
180 # We create and then move to make sure that when the file
180 # We create and then move to make sure that when the file
181 # appears to other processes, the buffer has the flushed
181 # appears to other processes, the buffer has the flushed
182 # and the file has been closed
182 # and the file has been closed
183 temp_furl_file = get_temp_furlfile(furl_file)
183 temp_furl_file = get_temp_furlfile(furl_file)
184 engine_tub.registerReference(ref, furlFile=temp_furl_file)
184 engine_tub.registerReference(ref, furlFile=temp_furl_file)
185 os.rename(temp_furl_file, furl_file)
185 os.rename(temp_furl_file, furl_file)
186
186
187 if location == '':
187 if location == '':
188 d = engine_tub.setLocationAutomatically()
188 d = engine_tub.setLocationAutomatically()
189 else:
189 else:
190 d = defer.maybeDeferred(engine_tub.setLocation, "%s:%i" % (location, engine_listener.getPortnum()))
190 d = defer.maybeDeferred(engine_tub.setLocation, "%s:%i" % (location, engine_listener.getPortnum()))
191
191
192 furl_file = config['controller']['engine_furl_file']
192 furl_file = config['controller']['engine_furl_file']
193 engine_fc_interface = import_item(config['controller']['engine_fc_interface'])
193 engine_fc_interface = import_item(config['controller']['engine_fc_interface'])
194 log.msg("Saving furl for the engine to file: %s" % furl_file)
194 log.msg("Saving furl for the engine to file: %s" % furl_file)
195 check_furl_file_security(furl_file, secure)
195 check_furl_file_security(furl_file, secure)
196 fc_controller = engine_fc_interface(controller_service)
196 fc_controller = engine_fc_interface(controller_service)
197 d.addCallback(register, fc_controller, furl_file=furl_file)
197 d.addCallback(register, fc_controller, furl_file=furl_file)
198
198
199 reactor.callWhenRunning(set_location_and_register)
199 reactor.callWhenRunning(set_location_and_register)
200 return engine_tub
200 return engine_tub
201
201
202 def start_controller():
202 def start_controller():
203 """
203 """
204 Start the controller by creating the service hierarchy and starting the reactor.
204 Start the controller by creating the service hierarchy and starting the reactor.
205
205
206 This method does the following:
206 This method does the following:
207
207
208 * It starts the controller logging
208 * It starts the controller logging
209 * In execute an import statement for the controller
209 * In execute an import statement for the controller
210 * It creates 2 `foolscap.Tub` instances for the client and the engines
210 * It creates 2 `foolscap.Tub` instances for the client and the engines
211 and registers `foolscap.Referenceables` with the tubs to expose the
211 and registers `foolscap.Referenceables` with the tubs to expose the
212 controller to engines and clients.
212 controller to engines and clients.
213 """
213 """
214 config = kernel_config_manager.get_config_obj()
214 config = kernel_config_manager.get_config_obj()
215
215
216 # Start logging
216 # Start logging
217 logfile = config['controller']['logfile']
217 logfile = config['controller']['logfile']
218 if logfile:
218 if logfile:
219 logfile = logfile + str(os.getpid()) + '.log'
219 logfile = logfile + str(os.getpid()) + '.log'
220 try:
220 try:
221 openLogFile = open(logfile, 'w')
221 openLogFile = open(logfile, 'w')
222 except:
222 except:
223 openLogFile = sys.stdout
223 openLogFile = sys.stdout
224 else:
224 else:
225 openLogFile = sys.stdout
225 openLogFile = sys.stdout
226 log.startLogging(openLogFile)
226 log.startLogging(openLogFile)
227
227
228 # Execute any user defined import statements
228 # Execute any user defined import statements
229 cis = config['controller']['import_statement']
229 cis = config['controller']['import_statement']
230 if cis:
230 if cis:
231 try:
231 try:
232 exec cis in globals(), locals()
232 exec cis in globals(), locals()
233 except:
233 except:
234 log.msg("Error running import_statement: %s" % cis)
234 log.msg("Error running import_statement: %s" % cis)
235
235
236 # Delete old furl files unless the reuse_furls is set
236 # Delete old furl files unless the reuse_furls is set
237 reuse = config['controller']['reuse_furls']
237 reuse = config['controller']['reuse_furls']
238 if not reuse:
238 if not reuse:
239 paths = (config['controller']['engine_furl_file'],
239 paths = (config['controller']['engine_furl_file'],
240 config['controller']['controller_interfaces']['task']['furl_file'],
240 config['controller']['controller_interfaces']['task']['furl_file'],
241 config['controller']['controller_interfaces']['multiengine']['furl_file']
241 config['controller']['controller_interfaces']['multiengine']['furl_file']
242 )
242 )
243 for p in paths:
243 for p in paths:
244 if os.path.isfile(p):
244 if os.path.isfile(p):
245 os.remove(p)
245 os.remove(p)
246
246
247 # Create the service hierarchy
247 # Create the service hierarchy
248 main_service = service.MultiService()
248 main_service = service.MultiService()
249 # The controller service
249 # The controller service
250 controller_service = controllerservice.ControllerService()
250 controller_service = controllerservice.ControllerService()
251 controller_service.setServiceParent(main_service)
251 controller_service.setServiceParent(main_service)
252 # The client tub and all its refereceables
252 # The client tub and all its refereceables
253 client_service = make_client_service(controller_service, config)
253 client_service = make_client_service(controller_service, config)
254 client_service.setServiceParent(main_service)
254 client_service.setServiceParent(main_service)
255 # The engine tub
255 # The engine tub
256 engine_service = make_engine_service(controller_service, config)
256 engine_service = make_engine_service(controller_service, config)
257 engine_service.setServiceParent(main_service)
257 engine_service.setServiceParent(main_service)
258 # Start the controller service and set things running
258 # Start the controller service and set things running
259 main_service.startService()
259 main_service.startService()
260 reactor.run()
260 reactor.run()
261
261
262 def init_config():
262 def init_config():
263 """
263 """
264 Initialize the configuration using default and command line options.
264 Initialize the configuration using default and command line options.
265 """
265 """
266
266
267 parser = OptionParser("""ipcontroller [options]
267 parser = OptionParser("""ipcontroller [options]
268
268
269 Start an IPython controller.
269 Start an IPython controller.
270
270
271 Use the IPYTHONDIR environment variable to change your IPython directory
271 Use the IPYTHONDIR environment variable to change your IPython directory
272 from the default of .ipython or _ipython. The log and security
272 from the default of .ipython or _ipython. The log and security
273 subdirectories of your IPython directory will be used by this script
273 subdirectories of your IPython directory will be used by this script
274 for log files and security files.""")
274 for log files and security files.""")
275
275
276 # Client related options
276 # Client related options
277 parser.add_option(
277 parser.add_option(
278 "--client-ip",
278 "--client-ip",
279 type="string",
279 type="string",
280 dest="client_ip",
280 dest="client_ip",
281 help="the IP address or hostname the controller will listen on for client connections"
281 help="the IP address or hostname the controller will listen on for client connections"
282 )
282 )
283 parser.add_option(
283 parser.add_option(
284 "--client-port",
284 "--client-port",
285 type="int",
285 type="int",
286 dest="client_port",
286 dest="client_port",
287 help="the port the controller will listen on for client connections"
287 help="the port the controller will listen on for client connections"
288 )
288 )
289 parser.add_option(
289 parser.add_option(
290 '--client-location',
290 '--client-location',
291 type="string",
291 type="string",
292 dest="client_location",
292 dest="client_location",
293 help="hostname or ip for clients to connect to"
293 help="hostname or ip for clients to connect to"
294 )
294 )
295 parser.add_option(
295 parser.add_option(
296 "-x",
296 "-x",
297 action="store_false",
297 action="store_false",
298 dest="client_secure",
298 dest="client_secure",
299 help="turn off all client security"
299 help="turn off all client security"
300 )
300 )
301 parser.add_option(
301 parser.add_option(
302 '--client-cert-file',
302 '--client-cert-file',
303 type="string",
303 type="string",
304 dest="client_cert_file",
304 dest="client_cert_file",
305 help="file to store the client SSL certificate"
305 help="file to store the client SSL certificate"
306 )
306 )
307 parser.add_option(
307 parser.add_option(
308 '--task-furl-file',
308 '--task-furl-file',
309 type="string",
309 type="string",
310 dest="task_furl_file",
310 dest="task_furl_file",
311 help="file to store the FURL for task clients to connect with"
311 help="file to store the FURL for task clients to connect with"
312 )
312 )
313 parser.add_option(
313 parser.add_option(
314 '--multiengine-furl-file',
314 '--multiengine-furl-file',
315 type="string",
315 type="string",
316 dest="multiengine_furl_file",
316 dest="multiengine_furl_file",
317 help="file to store the FURL for multiengine clients to connect with"
317 help="file to store the FURL for multiengine clients to connect with"
318 )
318 )
319 # Engine related options
319 # Engine related options
320 parser.add_option(
320 parser.add_option(
321 "--engine-ip",
321 "--engine-ip",
322 type="string",
322 type="string",
323 dest="engine_ip",
323 dest="engine_ip",
324 help="the IP address or hostname the controller will listen on for engine connections"
324 help="the IP address or hostname the controller will listen on for engine connections"
325 )
325 )
326 parser.add_option(
326 parser.add_option(
327 "--engine-port",
327 "--engine-port",
328 type="int",
328 type="int",
329 dest="engine_port",
329 dest="engine_port",
330 help="the port the controller will listen on for engine connections"
330 help="the port the controller will listen on for engine connections"
331 )
331 )
332 parser.add_option(
332 parser.add_option(
333 '--engine-location',
333 '--engine-location',
334 type="string",
334 type="string",
335 dest="engine_location",
335 dest="engine_location",
336 help="hostname or ip for engines to connect to"
336 help="hostname or ip for engines to connect to"
337 )
337 )
338 parser.add_option(
338 parser.add_option(
339 "-y",
339 "-y",
340 action="store_false",
340 action="store_false",
341 dest="engine_secure",
341 dest="engine_secure",
342 help="turn off all engine security"
342 help="turn off all engine security"
343 )
343 )
344 parser.add_option(
344 parser.add_option(
345 '--engine-cert-file',
345 '--engine-cert-file',
346 type="string",
346 type="string",
347 dest="engine_cert_file",
347 dest="engine_cert_file",
348 help="file to store the engine SSL certificate"
348 help="file to store the engine SSL certificate"
349 )
349 )
350 parser.add_option(
350 parser.add_option(
351 '--engine-furl-file',
351 '--engine-furl-file',
352 type="string",
352 type="string",
353 dest="engine_furl_file",
353 dest="engine_furl_file",
354 help="file to store the FURL for engines to connect with"
354 help="file to store the FURL for engines to connect with"
355 )
355 )
356 parser.add_option(
356 parser.add_option(
357 "-l", "--logfile",
357 "-l", "--logfile",
358 type="string",
358 type="string",
359 dest="logfile",
359 dest="logfile",
360 help="log file name (default is stdout)"
360 help="log file name (default is stdout)"
361 )
361 )
362 parser.add_option(
362 parser.add_option(
363 "-r",
363 "-r",
364 action="store_true",
364 action="store_true",
365 dest="reuse_furls",
365 dest="reuse_furls",
366 help="try to reuse all furl files"
366 help="try to reuse all furl files"
367 )
367 )
368
368
369 (options, args) = parser.parse_args()
369 (options, args) = parser.parse_args()
370
370
371 config = kernel_config_manager.get_config_obj()
371 config = kernel_config_manager.get_config_obj()
372
372
373 # Update with command line options
373 # Update with command line options
374 if options.client_ip is not None:
374 if options.client_ip is not None:
375 config['controller']['client_tub']['ip'] = options.client_ip
375 config['controller']['client_tub']['ip'] = options.client_ip
376 if options.client_port is not None:
376 if options.client_port is not None:
377 config['controller']['client_tub']['port'] = options.client_port
377 config['controller']['client_tub']['port'] = options.client_port
378 if options.client_location is not None:
378 if options.client_location is not None:
379 config['controller']['client_tub']['location'] = options.client_location
379 config['controller']['client_tub']['location'] = options.client_location
380 if options.client_secure is not None:
380 if options.client_secure is not None:
381 config['controller']['client_tub']['secure'] = options.client_secure
381 config['controller']['client_tub']['secure'] = options.client_secure
382 if options.client_cert_file is not None:
382 if options.client_cert_file is not None:
383 config['controller']['client_tub']['cert_file'] = options.client_cert_file
383 config['controller']['client_tub']['cert_file'] = options.client_cert_file
384 if options.task_furl_file is not None:
384 if options.task_furl_file is not None:
385 config['controller']['controller_interfaces']['task']['furl_file'] = options.task_furl_file
385 config['controller']['controller_interfaces']['task']['furl_file'] = options.task_furl_file
386 if options.multiengine_furl_file is not None:
386 if options.multiengine_furl_file is not None:
387 config['controller']['controller_interfaces']['multiengine']['furl_file'] = options.multiengine_furl_file
387 config['controller']['controller_interfaces']['multiengine']['furl_file'] = options.multiengine_furl_file
388 if options.engine_ip is not None:
388 if options.engine_ip is not None:
389 config['controller']['engine_tub']['ip'] = options.engine_ip
389 config['controller']['engine_tub']['ip'] = options.engine_ip
390 if options.engine_port is not None:
390 if options.engine_port is not None:
391 config['controller']['engine_tub']['port'] = options.engine_port
391 config['controller']['engine_tub']['port'] = options.engine_port
392 if options.engine_location is not None:
392 if options.engine_location is not None:
393 config['controller']['engine_tub']['location'] = options.engine_location
393 config['controller']['engine_tub']['location'] = options.engine_location
394 if options.engine_secure is not None:
394 if options.engine_secure is not None:
395 config['controller']['engine_tub']['secure'] = options.engine_secure
395 config['controller']['engine_tub']['secure'] = options.engine_secure
396 if options.engine_cert_file is not None:
396 if options.engine_cert_file is not None:
397 config['controller']['engine_tub']['cert_file'] = options.engine_cert_file
397 config['controller']['engine_tub']['cert_file'] = options.engine_cert_file
398 if options.engine_furl_file is not None:
398 if options.engine_furl_file is not None:
399 config['controller']['engine_furl_file'] = options.engine_furl_file
399 config['controller']['engine_furl_file'] = options.engine_furl_file
400 if options.reuse_furls is not None:
400 if options.reuse_furls is not None:
401 config['controller']['reuse_furls'] = options.reuse_furls
401 config['controller']['reuse_furls'] = options.reuse_furls
402
402
403 if options.logfile is not None:
403 if options.logfile is not None:
404 config['controller']['logfile'] = options.logfile
404 config['controller']['logfile'] = options.logfile
405
405
406 kernel_config_manager.update_config_obj(config)
406 kernel_config_manager.update_config_obj(config)
407
407
408 def main():
408 def main():
409 """
409 """
410 After creating the configuration information, start the controller.
410 After creating the configuration information, start the controller.
411 """
411 """
412 init_config()
412 init_config()
413 start_controller()
413 start_controller()
414
414
415 if __name__ == "__main__":
415 if __name__ == "__main__":
416 main()
416 main()
1 NO CONTENT: file renamed from scripts/iptest to IPython/scripts/iptest
NO CONTENT: file renamed from scripts/iptest to IPython/scripts/iptest
@@ -1,28 +1,28 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
2 # -*- coding: utf-8 -*-
3 """IPython -- An enhanced Interactive Python
3 """IPython -- An enhanced Interactive Python
4
4
5 This is just the startup wrapper script, kept deliberately to a minimum.
5 This is just the startup wrapper script, kept deliberately to a minimum.
6
6
7 The shell's mainloop() takes an optional argument, sys_exit (default=0). If
7 The shell's mainloop() takes an optional argument, sys_exit (default=0). If
8 set to 1, it calls sys.exit() at exit time. You can use the following code in
8 set to 1, it calls sys.exit() at exit time. You can use the following code in
9 your PYTHONSTARTUP file:
9 your PYTHONSTARTUP file:
10
10
11 import IPython
11 import IPython
12 IPython.Shell.IPShell().mainloop(sys_exit=1)
12 IPython.Shell.IPShell().mainloop(sys_exit=1)
13
13
14 [or simply IPython.Shell.IPShell().mainloop(1) ]
14 [or simply IPython.Shell.IPShell().mainloop(1) ]
15
15
16 and IPython will be your working environment when you start python. The final
16 and IPython will be your working environment when you start python. The final
17 sys.exit() call will make python exit transparently when IPython finishes, so
17 sys.exit() call will make python exit transparently when IPython finishes, so
18 you don't have an extra prompt to get out of.
18 you don't have an extra prompt to get out of.
19
19
20 This is probably useful to developers who manage multiple Python versions and
20 This is probably useful to developers who manage multiple Python versions and
21 don't want to have correspondingly multiple IPython versions. Note that in
21 don't want to have correspondingly multiple IPython versions. Note that in
22 this mode, there is no way to pass IPython any command-line options, as those
22 this mode, there is no way to pass IPython any command-line options, as those
23 are trapped first by Python itself.
23 are trapped first by Python itself.
24 """
24 """
25
25
26 import IPython.Shell
26 import IPython.core.shell
27
27
28 IPython.Shell.start().mainloop()
28 IPython.core.shell.start().mainloop()
1 NO CONTENT: file renamed from scripts/ipython-wx to IPython/scripts/ipython-wx
NO CONTENT: file renamed from scripts/ipython-wx to IPython/scripts/ipython-wx
1 NO CONTENT: file renamed from scripts/ipython_win_post_install.py to IPython/scripts/ipython_win_post_install.py
NO CONTENT: file renamed from scripts/ipython_win_post_install.py to IPython/scripts/ipython_win_post_install.py
1 NO CONTENT: file renamed from scripts/ipythonx to IPython/scripts/ipythonx
NO CONTENT: file renamed from scripts/ipythonx to IPython/scripts/ipythonx
@@ -1,9 +1,9 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2
2
3 """Thin wrapper around the IPython irunner module.
3 """Thin wrapper around the IPython irunner module.
4
4
5 Run with --help for details, or see the irunner source."""
5 Run with --help for details, or see the irunner source."""
6
6
7 from IPython import irunner
7 from IPython.lib import irunner
8
8
9 irunner.main()
9 irunner.main()
@@ -1,6 +1,6 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
2 # -*- coding: utf-8 -*-
3 """Simple wrapper around PyColorize, which colorizes python sources."""
3 """Simple wrapper around PyColorize, which colorizes python sources."""
4
4
5 import IPython.PyColorize
5 import IPython.utils.PyColorize
6 IPython.PyColorize.main()
6 IPython.utils.PyColorize.main()
@@ -1,32 +1,36 b''
1 include ipython.py
1 include ipython.py
2 include setupbase.py
2 include setupbase.py
3 include setupegg.py
3 include setupegg.py
4
4
5 graft scripts
6
7 graft setupext
5 graft setupext
8
6
9 graft IPython/UserConfig
10
11 graft IPython/kernel
7 graft IPython/kernel
12 graft IPython/config
8 graft IPython/config
9 graft IPython/core
10 graft IPython/deathrow
11 graft IPython/external
12 graft IPython/frontend
13 graft IPython/gui
14 graft IPython/lib
15 graft IPython/quarantine
16 graft IPython/scripts
13 graft IPython/testing
17 graft IPython/testing
14 graft IPython/tools
18 graft IPython/utils
15
19
16 recursive-include IPython/Extensions igrid_help*
20 recursive-include IPython/Extensions igrid_help*
17
21
18 graft docs
22 graft docs
19 exclude docs/\#*
23 exclude docs/\#*
20 exclude docs/man/*.1
24 exclude docs/man/*.1
21
25
22 # docs subdirs we want to skip
26 # docs subdirs we want to skip
23 prune docs/attic
27 prune docs/attic
24 prune docs/build
28 prune docs/build
25
29
26 global-exclude *~
30 global-exclude *~
27 global-exclude *.flc
31 global-exclude *.flc
28 global-exclude *.pyc
32 global-exclude *.pyc
29 global-exclude .dircopy.log
33 global-exclude .dircopy.log
30 global-exclude .svn
34 global-exclude .svn
31 global-exclude .bzr
35 global-exclude .bzr
32 global-exclude .hgignore
36 global-exclude .hgignore
@@ -1,191 +1,191 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 #
2 #
3 # IPython documentation build configuration file.
3 # IPython documentation build configuration file.
4
4
5 # NOTE: This file has been edited manually from the auto-generated one from
5 # NOTE: This file has been edited manually from the auto-generated one from
6 # sphinx. Do NOT delete and re-generate. If any changes from sphinx are
6 # sphinx. Do NOT delete and re-generate. If any changes from sphinx are
7 # needed, generate a scratch one and merge by hand any new fields needed.
7 # needed, generate a scratch one and merge by hand any new fields needed.
8
8
9 #
9 #
10 # This file is execfile()d with the current directory set to its containing dir.
10 # This file is execfile()d with the current directory set to its containing dir.
11 #
11 #
12 # The contents of this file are pickled, so don't put values in the namespace
12 # The contents of this file are pickled, so don't put values in the namespace
13 # that aren't pickleable (module imports are okay, they're removed automatically).
13 # that aren't pickleable (module imports are okay, they're removed automatically).
14 #
14 #
15 # All configuration values have a default value; values that are commented out
15 # All configuration values have a default value; values that are commented out
16 # serve to show the default value.
16 # serve to show the default value.
17
17
18 import sys, os
18 import sys, os
19
19
20 # If your extensions are in another directory, add it here. If the directory
20 # If your extensions are in another directory, add it here. If the directory
21 # is relative to the documentation root, use os.path.abspath to make it
21 # is relative to the documentation root, use os.path.abspath to make it
22 # absolute, like shown here.
22 # absolute, like shown here.
23 sys.path.append(os.path.abspath('../sphinxext'))
23 sys.path.append(os.path.abspath('../sphinxext'))
24
24
25 # Import support for ipython console session syntax highlighting (lives
25 # Import support for ipython console session syntax highlighting (lives
26 # in the sphinxext directory defined above)
26 # in the sphinxext directory defined above)
27 import ipython_console_highlighting
27 import ipython_console_highlighting
28
28
29 # We load the ipython release info into a dict by explicit execution
29 # We load the ipython release info into a dict by explicit execution
30 iprelease = {}
30 iprelease = {}
31 execfile('../../IPython/Release.py',iprelease)
31 execfile('../../IPython/core/release.py',iprelease)
32
32
33 # General configuration
33 # General configuration
34 # ---------------------
34 # ---------------------
35
35
36 # Add any Sphinx extension module names here, as strings. They can be extensions
36 # Add any Sphinx extension module names here, as strings. They can be extensions
37 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
37 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
38 extensions = ['sphinx.ext.autodoc',
38 extensions = ['sphinx.ext.autodoc',
39 'sphinx.ext.doctest',
39 'sphinx.ext.doctest',
40
40
41 'only_directives',
41 'only_directives',
42 'inheritance_diagram',
42 'inheritance_diagram',
43 'ipython_console_highlighting',
43 'ipython_console_highlighting',
44 # 'plot_directive', # disabled for now, needs matplotlib
44 # 'plot_directive', # disabled for now, needs matplotlib
45 'numpydoc', # to preprocess docstrings
45 'numpydoc', # to preprocess docstrings
46 ]
46 ]
47
47
48 # Add any paths that contain templates here, relative to this directory.
48 # Add any paths that contain templates here, relative to this directory.
49 templates_path = ['_templates']
49 templates_path = ['_templates']
50
50
51 # The suffix of source filenames.
51 # The suffix of source filenames.
52 source_suffix = '.txt'
52 source_suffix = '.txt'
53
53
54 # The master toctree document.
54 # The master toctree document.
55 master_doc = 'index'
55 master_doc = 'index'
56
56
57 # General substitutions.
57 # General substitutions.
58 project = 'IPython'
58 project = 'IPython'
59 copyright = '2008, The IPython Development Team'
59 copyright = '2008, The IPython Development Team'
60
60
61 # The default replacements for |version| and |release|, also used in various
61 # The default replacements for |version| and |release|, also used in various
62 # other places throughout the built documents.
62 # other places throughout the built documents.
63 #
63 #
64 # The full version, including alpha/beta/rc tags.
64 # The full version, including alpha/beta/rc tags.
65 release = iprelease['version']
65 release = iprelease['version']
66 # The short X.Y version.
66 # The short X.Y version.
67 version = '.'.join(release.split('.',2)[:2])
67 version = '.'.join(release.split('.',2)[:2])
68
68
69
69
70 # There are two options for replacing |today|: either, you set today to some
70 # There are two options for replacing |today|: either, you set today to some
71 # non-false value, then it is used:
71 # non-false value, then it is used:
72 #today = ''
72 #today = ''
73 # Else, today_fmt is used as the format for a strftime call.
73 # Else, today_fmt is used as the format for a strftime call.
74 today_fmt = '%B %d, %Y'
74 today_fmt = '%B %d, %Y'
75
75
76 # List of documents that shouldn't be included in the build.
76 # List of documents that shouldn't be included in the build.
77 #unused_docs = []
77 #unused_docs = []
78
78
79 # List of directories, relative to source directories, that shouldn't be searched
79 # List of directories, relative to source directories, that shouldn't be searched
80 # for source files.
80 # for source files.
81 exclude_dirs = ['attic']
81 exclude_dirs = ['attic']
82
82
83 # If true, '()' will be appended to :func: etc. cross-reference text.
83 # If true, '()' will be appended to :func: etc. cross-reference text.
84 #add_function_parentheses = True
84 #add_function_parentheses = True
85
85
86 # If true, the current module name will be prepended to all description
86 # If true, the current module name will be prepended to all description
87 # unit titles (such as .. function::).
87 # unit titles (such as .. function::).
88 #add_module_names = True
88 #add_module_names = True
89
89
90 # If true, sectionauthor and moduleauthor directives will be shown in the
90 # If true, sectionauthor and moduleauthor directives will be shown in the
91 # output. They are ignored by default.
91 # output. They are ignored by default.
92 #show_authors = False
92 #show_authors = False
93
93
94 # The name of the Pygments (syntax highlighting) style to use.
94 # The name of the Pygments (syntax highlighting) style to use.
95 pygments_style = 'sphinx'
95 pygments_style = 'sphinx'
96
96
97
97
98 # Options for HTML output
98 # Options for HTML output
99 # -----------------------
99 # -----------------------
100
100
101 # The style sheet to use for HTML and HTML Help pages. A file of that name
101 # The style sheet to use for HTML and HTML Help pages. A file of that name
102 # must exist either in Sphinx' static/ path, or in one of the custom paths
102 # must exist either in Sphinx' static/ path, or in one of the custom paths
103 # given in html_static_path.
103 # given in html_static_path.
104 html_style = 'default.css'
104 html_style = 'default.css'
105
105
106 # The name for this set of Sphinx documents. If None, it defaults to
106 # The name for this set of Sphinx documents. If None, it defaults to
107 # "<project> v<release> documentation".
107 # "<project> v<release> documentation".
108 #html_title = None
108 #html_title = None
109
109
110 # The name of an image file (within the static path) to place at the top of
110 # The name of an image file (within the static path) to place at the top of
111 # the sidebar.
111 # the sidebar.
112 #html_logo = None
112 #html_logo = None
113
113
114 # Add any paths that contain custom static files (such as style sheets) here,
114 # Add any paths that contain custom static files (such as style sheets) here,
115 # relative to this directory. They are copied after the builtin static files,
115 # relative to this directory. They are copied after the builtin static files,
116 # so a file named "default.css" will overwrite the builtin "default.css".
116 # so a file named "default.css" will overwrite the builtin "default.css".
117 html_static_path = ['_static']
117 html_static_path = ['_static']
118
118
119 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
119 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
120 # using the given strftime format.
120 # using the given strftime format.
121 html_last_updated_fmt = '%b %d, %Y'
121 html_last_updated_fmt = '%b %d, %Y'
122
122
123 # If true, SmartyPants will be used to convert quotes and dashes to
123 # If true, SmartyPants will be used to convert quotes and dashes to
124 # typographically correct entities.
124 # typographically correct entities.
125 #html_use_smartypants = True
125 #html_use_smartypants = True
126
126
127 # Custom sidebar templates, maps document names to template names.
127 # Custom sidebar templates, maps document names to template names.
128 #html_sidebars = {}
128 #html_sidebars = {}
129
129
130 # Additional templates that should be rendered to pages, maps page names to
130 # Additional templates that should be rendered to pages, maps page names to
131 # template names.
131 # template names.
132 #html_additional_pages = {}
132 #html_additional_pages = {}
133
133
134 # If false, no module index is generated.
134 # If false, no module index is generated.
135 #html_use_modindex = True
135 #html_use_modindex = True
136
136
137 # If true, the reST sources are included in the HTML build as _sources/<name>.
137 # If true, the reST sources are included in the HTML build as _sources/<name>.
138 #html_copy_source = True
138 #html_copy_source = True
139
139
140 # If true, an OpenSearch description file will be output, and all pages will
140 # If true, an OpenSearch description file will be output, and all pages will
141 # contain a <link> tag referring to it. The value of this option must be the
141 # contain a <link> tag referring to it. The value of this option must be the
142 # base URL from which the finished HTML is served.
142 # base URL from which the finished HTML is served.
143 #html_use_opensearch = ''
143 #html_use_opensearch = ''
144
144
145 # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
145 # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
146 #html_file_suffix = ''
146 #html_file_suffix = ''
147
147
148 # Output file base name for HTML help builder.
148 # Output file base name for HTML help builder.
149 htmlhelp_basename = 'ipythondoc'
149 htmlhelp_basename = 'ipythondoc'
150
150
151
151
152 # Options for LaTeX output
152 # Options for LaTeX output
153 # ------------------------
153 # ------------------------
154
154
155 # The paper size ('letter' or 'a4').
155 # The paper size ('letter' or 'a4').
156 latex_paper_size = 'letter'
156 latex_paper_size = 'letter'
157
157
158 # The font size ('10pt', '11pt' or '12pt').
158 # The font size ('10pt', '11pt' or '12pt').
159 latex_font_size = '11pt'
159 latex_font_size = '11pt'
160
160
161 # Grouping the document tree into LaTeX files. List of tuples
161 # Grouping the document tree into LaTeX files. List of tuples
162 # (source start file, target name, title, author, document class [howto/manual]).
162 # (source start file, target name, title, author, document class [howto/manual]).
163
163
164 latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation',
164 latex_documents = [ ('index', 'ipython.tex', 'IPython Documentation',
165 ur"""The IPython Development Team""",
165 ur"""The IPython Development Team""",
166 'manual'),
166 'manual'),
167 ]
167 ]
168
168
169 # The name of an image file (relative to this directory) to place at the top of
169 # The name of an image file (relative to this directory) to place at the top of
170 # the title page.
170 # the title page.
171 #latex_logo = None
171 #latex_logo = None
172
172
173 # For "manual" documents, if this is true, then toplevel headings are parts,
173 # For "manual" documents, if this is true, then toplevel headings are parts,
174 # not chapters.
174 # not chapters.
175 #latex_use_parts = False
175 #latex_use_parts = False
176
176
177 # Additional stuff for the LaTeX preamble.
177 # Additional stuff for the LaTeX preamble.
178 #latex_preamble = ''
178 #latex_preamble = ''
179
179
180 # Documents to append as an appendix to all manuals.
180 # Documents to append as an appendix to all manuals.
181 #latex_appendices = []
181 #latex_appendices = []
182
182
183 # If false, no module index is generated.
183 # If false, no module index is generated.
184 #latex_use_modindex = True
184 #latex_use_modindex = True
185
185
186
186
187 # Cleanup
187 # Cleanup
188 # -------
188 # -------
189 # delete release info to avoid pickling errors from sphinx
189 # delete release info to avoid pickling errors from sphinx
190
190
191 del iprelease
191 del iprelease
@@ -1,261 +1,266 b''
1 =============================
1 =============================
2 IPython module reorganization
2 IPython module reorganization
3 =============================
3 =============================
4
4
5 Currently, IPython has many top-level modules that serve many different purposes.
5 Currently, IPython has many top-level modules that serve many different purposes.
6 The lack of organization make it very difficult for developers to work on IPython
6 The lack of organization make it very difficult for developers to work on IPython
7 and understand its design. This document contains notes about how we will reorganize
7 and understand its design. This document contains notes about how we will reorganize
8 the modules into sub-packages.
8 the modules into sub-packages.
9
9
10 .. warning::
10 .. warning::
11
11
12 This effort will possibly break third party packages that use IPython as
12 This effort will possibly break third party packages that use IPython as
13 a library or hack on the IPython internals.
13 a library or hack on the IPython internals.
14
14
15 .. warning::
15 .. warning::
16
16
17 This effort will result in the removal from IPython of certain modules
17 This effort will result in the removal from IPython of certain modules
18 that are not used anymore, don't currently work, are unmaintained, etc.
18 that are not used anymore, don't currently work, are unmaintained, etc.
19
19
20
20
21 Current subpackges
21 Current subpackges
22 ==================
22 ==================
23
23
24 IPython currently has the following sub-packages:
24 IPython currently has the following sub-packages:
25
25
26 * :mod:`IPython.config`
26 * :mod:`IPython.config`
27
27
28 * :mod:`IPython.Extensions`
28 * :mod:`IPython.Extensions`
29
29
30 * :mod:`IPython.external`
30 * :mod:`IPython.external`
31
31
32 * :mod:`IPython.frontend`
32 * :mod:`IPython.frontend`
33
33
34 * :mod:`IPython.gui`
34 * :mod:`IPython.gui`
35
35
36 * :mod:`IPython.kernel`
36 * :mod:`IPython.kernel`
37
37
38 * :mod:`IPython.testing`
38 * :mod:`IPython.testing`
39
39
40 * :mod:`IPython.tests`
40 * :mod:`IPython.tests`
41
41
42 * :mod:`IPython.tools`
42 * :mod:`IPython.tools`
43
43
44 * :mod:`IPython.UserConfig`
44 * :mod:`IPython.UserConfig`
45
45
46 New Subpackages to be created
46 New Subpackages to be created
47 =============================
47 =============================
48
48
49 We propose to create the following new sub-packages:
49 We propose to create the following new sub-packages:
50
50
51 * :mod:`IPython.core`. This sub-package will contain the core of the IPython
51 * :mod:`IPython.core`. This sub-package will contain the core of the IPython
52 interpreter, but none of its extended capabilities.
52 interpreter, but none of its extended capabilities.
53
53
54 * :mod:`IPython.lib`. IPython has many extended capabilities that are not part
54 * :mod:`IPython.lib`. IPython has many extended capabilities that are not part
55 of the IPython core. These things will go here. Any better names than
55 of the IPython core. These things will go here. Any better names than
56 :mod:`IPython.lib`?
56 :mod:`IPython.lib`?
57
57
58 * :mod:`IPython.utils`. This sub-package will contain anything that might
58 * :mod:`IPython.utils`. This sub-package will contain anything that might
59 eventually be found in the Python standard library, like things in
59 eventually be found in the Python standard library, like things in
60 :mod:`genutils`. Each sub-module in this sub-package should contain
60 :mod:`genutils`. Each sub-module in this sub-package should contain
61 functions and classes that serve a single purpose.
61 functions and classes that serve a single purpose.
62
62
63 * :mod:`IPython.deathrow`. This is for code that is untested and/or rotting
63 * :mod:`IPython.deathrow`. This is for code that is untested and/or rotting
64 and needs to be removed from IPython. Eventually all this code will either
64 and needs to be removed from IPython. Eventually all this code will either
65 i) be revived by someone willing to maintain it with tests and docs and
65 i) be revived by someone willing to maintain it with tests and docs and
66 re-included into IPython or 2) be removed from IPython proper, but put into
66 re-included into IPython or 2) be removed from IPython proper, but put into
67 a separate top-level (not IPython) package that we keep around. No new code
67 a separate top-level (not IPython) package that we keep around. No new code
68 will be allowed here.
68 will be allowed here.
69
69
70 * :mod:`IPython.quarantine`. This is for code that doesn't meet IPython's
70 * :mod:`IPython.quarantine`. This is for code that doesn't meet IPython's
71 standards, but that we plan on keeping. To be moved out of this sub-package
71 standards, but that we plan on keeping. To be moved out of this sub-package
72 a module needs to have a maintainer, tests and documentation.
72 a module needs to have a maintainer, tests and documentation.
73
73
74 Prodecure
74 Prodecure
75 =========
75 =========
76
76
77 1. Move the file to its new location with its new name.
77 1. Move the file to its new location with its new name.
78 2. Rename all import statements to reflect the change.
78 2. Rename all import statements to reflect the change.
79 3. Run PyFlakes on each changes module.
79 3. Run PyFlakes on each changes module.
80 3. Add tests/test_imports.py to test it.
80 3. Add tests/test_imports.py to test it.
81
81
82 Need to modify iptests to properly skip modules that are no longer top
82 Need to modify iptests to properly skip modules that are no longer top
83 level modules.
83 level modules.
84
84
85 Need to update the top level IPython/__init__.py file.
85 Need to update the top level IPython/__init__.py file.
86
86
87 Need to get installation working correctly.
88
89 When running python setup.py sdist, the Sphinx API docs fail to build because
90 of something going on with IPython.core.fakemodule
91
87 Where things will be moved
92 Where things will be moved
88 ==========================
93 ==========================
89
94
90 Top-level modules:
95 Top-level modules:
91
96
92 * :file:`background_jobs.py`. Move to :file:`IPython/lib/backgroundjobs.py`.
97 * :file:`background_jobs.py`. Move to :file:`IPython/lib/backgroundjobs.py`.
93
98
94 * :file:`ColorANSI.py`. Move to :file:`IPython/utils/coloransi.py`.
99 * :file:`ColorANSI.py`. Move to :file:`IPython/utils/coloransi.py`.
95
100
96 * :file:`completer.py`. Move to :file:`IPython/core/completer.py`.
101 * :file:`completer.py`. Move to :file:`IPython/core/completer.py`.
97
102
98 * :file:`ConfigLoader.py`. Move to :file:`IPython/config/configloader.py`.
103 * :file:`ConfigLoader.py`. Move to :file:`IPython/config/configloader.py`.
99
104
100 * :file:`CrashHandler.py`. Move to :file:`IPython/core/crashhandler`.
105 * :file:`CrashHandler.py`. Move to :file:`IPython/core/crashhandler`.
101
106
102 * :file:`Debugger.py`. Move to :file:`IPython/core/debugger.py`.
107 * :file:`Debugger.py`. Move to :file:`IPython/core/debugger.py`.
103
108
104 * :file:`deep_reload.py`. Move to :file:`IPython/lib/deepreload.py`.
109 * :file:`deep_reload.py`. Move to :file:`IPython/lib/deepreload.py`.
105
110
106 * :file:`demo.py`. Move to :file:`IPython/lib/demo.py`.
111 * :file:`demo.py`. Move to :file:`IPython/lib/demo.py`.
107
112
108 * :file:`DPyGetOpt.py`. Move to :mod:`IPython.utils` and replace with newer options parser.
113 * :file:`DPyGetOpt.py`. Move to :mod:`IPython.utils` and replace with newer options parser.
109
114
110 * :file:`dtutils.py`. Move to :file:`IPython.deathrow`.
115 * :file:`dtutils.py`. Move to :file:`IPython.deathrow`.
111
116
112 * :file:`excolors.py`. Move to :file:`IPython.core` or :file:`IPython.config`.
117 * :file:`excolors.py`. Move to :file:`IPython.core` or :file:`IPython.config`.
113 Maybe move to :mod:`IPython.lib` or :mod:`IPython.python`?
118 Maybe move to :mod:`IPython.lib` or :mod:`IPython.python`?
114
119
115 * :file:`FakeModule.py`. Move to :file:`IPython/core/fakemodule.py`.
120 * :file:`FakeModule.py`. Move to :file:`IPython/core/fakemodule.py`.
116
121
117 * :file:`generics.py`. Move to :file:`IPython.python`.
122 * :file:`generics.py`. Move to :file:`IPython.python`.
118
123
119 * :file:`genutils.py`. Move to :file:`IPython.utils`.
124 * :file:`genutils.py`. Move to :file:`IPython.utils`.
120
125
121 * :file:`Gnuplot2.py`. Move to :file:`IPython.sandbox`.
126 * :file:`Gnuplot2.py`. Move to :file:`IPython.sandbox`.
122
127
123 * :file:`GnuplotInteractive.py`. Move to :file:`IPython.sandbox`.
128 * :file:`GnuplotInteractive.py`. Move to :file:`IPython.sandbox`.
124
129
125 * :file:`GnuplotRuntime.py`. Move to :file:`IPython.sandbox`.
130 * :file:`GnuplotRuntime.py`. Move to :file:`IPython.sandbox`.
126
131
127 * :file:`numutils.py`. Move to :file:`IPython.sandbox`.
132 * :file:`numutils.py`. Move to :file:`IPython.sandbox`.
128
133
129 * :file:`twshell.py`. Move to :file:`IPython.sandbox`.
134 * :file:`twshell.py`. Move to :file:`IPython.sandbox`.
130
135
131 * :file:`Extensions`. This needs to be gone through separately. Minimally,
136 * :file:`Extensions`. This needs to be gone through separately. Minimally,
132 the package should be renamed to :file:`extensions`.
137 the package should be renamed to :file:`extensions`.
133
138
134 * :file:`history.py`. Move to :file:`IPython.core`.
139 * :file:`history.py`. Move to :file:`IPython.core`.
135
140
136 * :file:`hooks.py`. Move to :file:`IPython.core`.
141 * :file:`hooks.py`. Move to :file:`IPython.core`.
137
142
138 * :file:`ipapi.py`. Move to :file:`IPython.core`.
143 * :file:`ipapi.py`. Move to :file:`IPython.core`.
139
144
140 * :file:`iplib.py`. Move to :file:`IPython.core`.
145 * :file:`iplib.py`. Move to :file:`IPython.core`.
141
146
142 * :file:`ipmaker.py`: Move to :file:`IPython.core`.
147 * :file:`ipmaker.py`: Move to :file:`IPython.core`.
143
148
144 * :file:`ipstruct.py`. Move to :file:`IPython.python`.
149 * :file:`ipstruct.py`. Move to :file:`IPython.python`.
145
150
146 * :file:`irunner.py`. Move to :file:`IPython.scripts`. ???
151 * :file:`irunner.py`. Move to :file:`IPython.scripts`. ???
147
152
148 * :file:`Itpl.py`. Move to :file:`deathrow/Itpl.py`. Copy already in
153 * :file:`Itpl.py`. Move to :file:`deathrow/Itpl.py`. Copy already in
149 :file:`IPython.external`.
154 :file:`IPython.external`.
150
155
151 * :file:`Logger.py`. Move to :file:`IPython/core/logger.py`.
156 * :file:`Logger.py`. Move to :file:`IPython/core/logger.py`.
152
157
153 * :file:`macro.py`. Move to :file:`IPython.core`.
158 * :file:`macro.py`. Move to :file:`IPython.core`.
154
159
155 * :file:`Magic.py`. Move to :file:`IPython/core/magic.py`.
160 * :file:`Magic.py`. Move to :file:`IPython/core/magic.py`.
156
161
157 * :file:`OInspect.py`. Move to :file:`IPython/core/oinspect.py`.
162 * :file:`OInspect.py`. Move to :file:`IPython/core/oinspect.py`.
158
163
159 * :file:`OutputTrap.py`. Move to :file:`IPython/core/outputtrap.py`.
164 * :file:`OutputTrap.py`. Move to :file:`IPython/core/outputtrap.py`.
160
165
161 * :file:`platutils.py`. Move to :file:`IPython.python`.
166 * :file:`platutils.py`. Move to :file:`IPython.python`.
162
167
163 * :file:`platutils_dummy.py`. Move to :file:`IPython.python`.
168 * :file:`platutils_dummy.py`. Move to :file:`IPython.python`.
164
169
165 * :file:`platutils_posix.py`. Move to :file:`IPython.python`.
170 * :file:`platutils_posix.py`. Move to :file:`IPython.python`.
166
171
167 * :file:`platutils_win32.py`. Move to :file:`IPython.python`.
172 * :file:`platutils_win32.py`. Move to :file:`IPython.python`.
168
173
169 * :file:`prefilter.py`: Move to :file:`IPython.core`.
174 * :file:`prefilter.py`: Move to :file:`IPython.core`.
170
175
171 * :file:`Prompts.py`. Move to :file:`IPython/core/prompts.py` or
176 * :file:`Prompts.py`. Move to :file:`IPython/core/prompts.py` or
172 :file:`IPython/frontend/prompts.py`.
177 :file:`IPython/frontend/prompts.py`.
173
178
174 * :file:`PyColorize.py`. Replace with pygments? If not, move to
179 * :file:`PyColorize.py`. Replace with pygments? If not, move to
175 :file:`IPython/core/pycolorize.py`. Maybe move to :mod:`IPython.lib` or
180 :file:`IPython/core/pycolorize.py`. Maybe move to :mod:`IPython.lib` or
176 :mod:`IPython.python`?
181 :mod:`IPython.python`?
177
182
178 * :file:`Release.py`. Move to ??? or remove?
183 * :file:`Release.py`. Move to ??? or remove?
179
184
180 * :file:`rlineimpl.py`. Move to :file:`IPython.core`.
185 * :file:`rlineimpl.py`. Move to :file:`IPython.core`.
181
186
182 * :file:`shadowns.py`. Move to :file:`IPython.core`.
187 * :file:`shadowns.py`. Move to :file:`IPython.core`.
183
188
184 * :file:`Shell.py`. Move to :file:`IPython.core.shell.py` or
189 * :file:`Shell.py`. Move to :file:`IPython.core.shell.py` or
185 :file:`IPython/frontend/shell.py`.
190 :file:`IPython/frontend/shell.py`.
186
191
187 * :file:`shellglobals.py`. Move to :file:`IPython.core`.
192 * :file:`shellglobals.py`. Move to :file:`IPython.core`.
188
193
189 * :file:`strdispatch.py`. Move to :file:`IPython.python`.
194 * :file:`strdispatch.py`. Move to :file:`IPython.python`.
190
195
191 * :file:`twshell.py`. Move to :file:`IPython.sandbox`.
196 * :file:`twshell.py`. Move to :file:`IPython.sandbox`.
192
197
193 * :file:`ultraTB.py`. Move to :file:`IPython/core/ultratb.py`.
198 * :file:`ultraTB.py`. Move to :file:`IPython/core/ultratb.py`.
194
199
195 * :file:`upgrade_dir.py`. Move to :file:`IPython/utils/upgradedir.py`.
200 * :file:`upgrade_dir.py`. Move to :file:`IPython/utils/upgradedir.py`.
196
201
197 * :file:`usage.py`. Move to :file:`IPython.core`.
202 * :file:`usage.py`. Move to :file:`IPython.core`.
198
203
199 * :file:`wildcard.py`. Move to :file:`IPython.utils`.
204 * :file:`wildcard.py`. Move to :file:`IPython.utils`.
200
205
201 * :file:`winconsole.py`. Move to :file:`IPython.utils`.
206 * :file:`winconsole.py`. Move to :file:`IPython.utils`.
202
207
203 Top-level sub-packages:
208 Top-level sub-packages:
204
209
205 * :file:`testing`. Good where it is.
210 * :file:`testing`. Good where it is.
206
211
207 * :file:`tests`. Remove.
212 * :file:`tests`. Remove.
208
213
209 * :file:`tools`. Things in here need to be looked at and moved elsewhere like
214 * :file:`tools`. Things in here need to be looked at and moved elsewhere like
210 :file:`IPython.utils`.
215 :file:`IPython.utils`.
211
216
212 * :file:`UserConfig`. Move to :file:`IPython.config.userconfig`.
217 * :file:`UserConfig`. Move to :file:`IPython.config.userconfig`.
213
218
214 * :file:`config`. Good where it is!
219 * :file:`config`. Good where it is!
215
220
216 * :file:`external`. Good where it is!
221 * :file:`external`. Good where it is!
217
222
218 * :file:`frontend`. Good where it is!
223 * :file:`frontend`. Good where it is!
219
224
220 * :file:`gui`. Eventually this should be moved to a subdir of
225 * :file:`gui`. Eventually this should be moved to a subdir of
221 :file:`IPython.frontend`.
226 :file:`IPython.frontend`.
222
227
223 * :file:`kernel`. Good where it is.
228 * :file:`kernel`. Good where it is.
224
229
225
230
226
231
227
232
228
233
229
234
230
235
231
236
232
237
233
238
234
239
235
240
236
241
237
242
238
243
239
244
240
245
241 Other things
246 Other things
242 ============
247 ============
243
248
244 When these files are moved around, a number of other things will happen at the same time:
249 When these files are moved around, a number of other things will happen at the same time:
245
250
246 1. Test files will be created for each module in IPython. Minimally, all
251 1. Test files will be created for each module in IPython. Minimally, all
247 modules will be imported as a part of the test. This will serve as a
252 modules will be imported as a part of the test. This will serve as a
248 test of the module reorganization. These tests will be put into new
253 test of the module reorganization. These tests will be put into new
249 :file:`tests` subdirectories that each package will have.
254 :file:`tests` subdirectories that each package will have.
250
255
251 2. PyFlakes and other code checkers will be run to look for problems.
256 2. PyFlakes and other code checkers will be run to look for problems.
252
257
253 3. Modules will be renamed to comply with PEP 8 naming conventions: all
258 3. Modules will be renamed to comply with PEP 8 naming conventions: all
254 lowercase and no special characters like ``-`` or ``_``.
259 lowercase and no special characters like ``-`` or ``_``.
255
260
256 4. Existing tests will be moved to the appropriate :file:`tests`
261 4. Existing tests will be moved to the appropriate :file:`tests`
257 subdirectories.
262 subdirectories.
258
263
259
264
260
265
261
266
@@ -1,11 +1,11 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
2 # -*- coding: utf-8 -*-
3 """IPython -- An enhanced Interactive Python
3 """IPython -- An enhanced Interactive Python
4
4
5 The actual ipython script to be installed with 'python setup.py install' is
5 The actual ipython script to be installed with 'python setup.py install' is
6 in './scripts' directory. This file is here (ipython source root directory)
6 in './scripts' directory. This file is here (ipython source root directory)
7 to facilitate non-root 'zero-installation' (just copy the source tree
7 to facilitate non-root 'zero-installation' (just copy the source tree
8 somewhere and run ipython.py) and development. """
8 somewhere and run ipython.py) and development. """
9
9
10 import IPython.Shell
10 import IPython.core.shell
11 IPython.Shell.start().mainloop()
11 IPython.core.shell.start().mainloop()
@@ -1,189 +1,190 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
2 # -*- coding: utf-8 -*-
3 """Setup script for IPython.
3 """Setup script for IPython.
4
4
5 Under Posix environments it works like a typical setup.py script.
5 Under Posix environments it works like a typical setup.py script.
6 Under Windows, the command sdist is not supported, since IPython
6 Under Windows, the command sdist is not supported, since IPython
7 requires utilities which are not available under Windows."""
7 requires utilities which are not available under Windows."""
8
8
9 #-------------------------------------------------------------------------------
9 #-------------------------------------------------------------------------------
10 # Copyright (C) 2008 The IPython Development Team
10 # Copyright (C) 2008 The IPython Development Team
11 #
11 #
12 # Distributed under the terms of the BSD License. The full license is in
12 # Distributed under the terms of the BSD License. The full license is in
13 # the file COPYING, distributed as part of this software.
13 # the file COPYING, distributed as part of this software.
14 #-------------------------------------------------------------------------------
14 #-------------------------------------------------------------------------------
15
15
16 #-------------------------------------------------------------------------------
16 #-------------------------------------------------------------------------------
17 # Imports
17 # Imports
18 #-------------------------------------------------------------------------------
18 #-------------------------------------------------------------------------------
19
19
20 # Stdlib imports
20 # Stdlib imports
21 import os
21 import os
22 import sys
22 import sys
23
23
24 from glob import glob
24 from glob import glob
25
25
26 # BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
26 # BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
27 # update it when the contents of directories change.
27 # update it when the contents of directories change.
28 if os.path.exists('MANIFEST'): os.remove('MANIFEST')
28 if os.path.exists('MANIFEST'): os.remove('MANIFEST')
29
29
30 from distutils.core import setup
30 from distutils.core import setup
31
31
32 # Local imports
33 from IPython.utils.genutils import target_update
32 from IPython.utils.genutils import target_update
34
33
35 from setupbase import (
34 from setupbase import (
36 setup_args,
35 setup_args,
37 find_packages,
36 find_packages,
38 find_package_data,
37 find_package_data,
39 find_scripts,
38 find_scripts,
40 find_data_files,
39 find_data_files,
41 check_for_dependencies
40 check_for_dependencies
42 )
41 )
43
42
44 isfile = os.path.isfile
43 isfile = os.path.isfile
44 pjoin = os.path.join
45
45
46 #-------------------------------------------------------------------------------
46 #-------------------------------------------------------------------------------
47 # Handle OS specific things
47 # Handle OS specific things
48 #-------------------------------------------------------------------------------
48 #-------------------------------------------------------------------------------
49
49
50 if os.name == 'posix':
50 if os.name == 'posix':
51 os_name = 'posix'
51 os_name = 'posix'
52 elif os.name in ['nt','dos']:
52 elif os.name in ['nt','dos']:
53 os_name = 'windows'
53 os_name = 'windows'
54 else:
54 else:
55 print 'Unsupported operating system:',os.name
55 print 'Unsupported operating system:',os.name
56 sys.exit(1)
56 sys.exit(1)
57
57
58 # Under Windows, 'sdist' has not been supported. Now that the docs build with
58 # Under Windows, 'sdist' has not been supported. Now that the docs build with
59 # Sphinx it might work, but let's not turn it on until someone confirms that it
59 # Sphinx it might work, but let's not turn it on until someone confirms that it
60 # actually works.
60 # actually works.
61 if os_name == 'windows' and 'sdist' in sys.argv:
61 if os_name == 'windows' and 'sdist' in sys.argv:
62 print 'The sdist command is not available under Windows. Exiting.'
62 print 'The sdist command is not available under Windows. Exiting.'
63 sys.exit(1)
63 sys.exit(1)
64
64
65 #-------------------------------------------------------------------------------
65 #-------------------------------------------------------------------------------
66 # Things related to the IPython documentation
66 # Things related to the IPython documentation
67 #-------------------------------------------------------------------------------
67 #-------------------------------------------------------------------------------
68
68
69 # update the manuals when building a source dist
69 # update the manuals when building a source dist
70 if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
70 if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
71 import textwrap
71 import textwrap
72
72
73 # List of things to be updated. Each entry is a triplet of args for
73 # List of things to be updated. Each entry is a triplet of args for
74 # target_update()
74 # target_update()
75 to_update = [
75 to_update = [
76 # FIXME - Disabled for now: we need to redo an automatic way
76 # FIXME - Disabled for now: we need to redo an automatic way
77 # of generating the magic info inside the rst.
77 # of generating the magic info inside the rst.
78 #('docs/magic.tex',
78 #('docs/magic.tex',
79 #['IPython/Magic.py'],
79 #['IPython/Magic.py'],
80 #"cd doc && ./update_magic.sh" ),
80 #"cd doc && ./update_magic.sh" ),
81
81
82 ('docs/man/ipython.1.gz',
82 ('docs/man/ipython.1.gz',
83 ['docs/man/ipython.1'],
83 ['docs/man/ipython.1'],
84 "cd docs/man && gzip -9c ipython.1 > ipython.1.gz"),
84 "cd docs/man && gzip -9c ipython.1 > ipython.1.gz"),
85
85
86 ('docs/man/pycolor.1.gz',
86 ('docs/man/pycolor.1.gz',
87 ['docs/man/pycolor.1'],
87 ['docs/man/pycolor.1'],
88 "cd docs/man && gzip -9c pycolor.1 > pycolor.1.gz"),
88 "cd docs/man && gzip -9c pycolor.1 > pycolor.1.gz"),
89 ]
89 ]
90
90
91 # Only build the docs if sphinx is present
91 # Only build the docs if sphinx is present
92 try:
92 try:
93 import sphinx
93 import sphinx
94 except ImportError:
94 except ImportError:
95 pass
95 pass
96 else:
96 else:
97 # The Makefile calls the do_sphinx scripts to build html and pdf, so
97 # The Makefile calls the do_sphinx scripts to build html and pdf, so
98 # just one target is enough to cover all manual generation
98 # just one target is enough to cover all manual generation
99
99
100 # First, compute all the dependencies that can force us to rebuild the
100 # First, compute all the dependencies that can force us to rebuild the
101 # docs. Start with the main release file that contains metadata
101 # docs. Start with the main release file that contains metadata
102 docdeps = ['IPython/Release.py']
102 docdeps = ['IPython/core/release.py']
103 # Inculde all the reST sources
103 # Inculde all the reST sources
104 pjoin = os.path.join
104 pjoin = os.path.join
105 for dirpath,dirnames,filenames in os.walk('docs/source'):
105 for dirpath,dirnames,filenames in os.walk('docs/source'):
106 if dirpath in ['_static','_templates']:
106 if dirpath in ['_static','_templates']:
107 continue
107 continue
108 docdeps += [ pjoin(dirpath,f) for f in filenames
108 docdeps += [ pjoin(dirpath,f) for f in filenames
109 if f.endswith('.txt') ]
109 if f.endswith('.txt') ]
110 # and the examples
110 # and the examples
111 for dirpath,dirnames,filenames in os.walk('docs/example'):
111 for dirpath,dirnames,filenames in os.walk('docs/example'):
112 docdeps += [ pjoin(dirpath,f) for f in filenames
112 docdeps += [ pjoin(dirpath,f) for f in filenames
113 if not f.endswith('~') ]
113 if not f.endswith('~') ]
114 # then, make them all dependencies for the main PDF (the html will get
114 # then, make them all dependencies for the main PDF (the html will get
115 # auto-generated as well).
115 # auto-generated as well).
116 to_update.append(
116 to_update.append(
117 ('docs/dist/ipython.pdf',
117 ('docs/dist/ipython.pdf',
118 docdeps,
118 docdeps,
119 "cd docs && make dist")
119 "cd docs && make dist")
120 )
120 )
121
121
122 [ target_update(*t) for t in to_update ]
122 [ target_update(*t) for t in to_update ]
123
123
124
124
125 #---------------------------------------------------------------------------
125 #---------------------------------------------------------------------------
126 # Find all the packages, package data, scripts and data_files
126 # Find all the packages, package data, scripts and data_files
127 #---------------------------------------------------------------------------
127 #---------------------------------------------------------------------------
128
128
129 packages = find_packages()
129 packages = find_packages()
130 package_data = find_package_data()
130 package_data = find_package_data()
131 scripts = find_scripts()
131 scripts = find_scripts()
132 data_files = find_data_files()
132 data_files = find_data_files()
133
133
134 #---------------------------------------------------------------------------
134 #---------------------------------------------------------------------------
135 # Handle dependencies and setuptools specific things
135 # Handle dependencies and setuptools specific things
136 #---------------------------------------------------------------------------
136 #---------------------------------------------------------------------------
137
137
138 # This dict is used for passing extra arguments that are setuptools
138 # This dict is used for passing extra arguments that are setuptools
139 # specific to setup
139 # specific to setup
140 setuptools_extra_args = {}
140 setuptools_extra_args = {}
141
141
142 if 'setuptools' in sys.modules:
142 if 'setuptools' in sys.modules:
143 setuptools_extra_args['zip_safe'] = False
143 setuptools_extra_args['zip_safe'] = False
144 setuptools_extra_args['entry_points'] = {
144 setuptools_extra_args['entry_points'] = {
145 'console_scripts': [
145 'console_scripts': [
146 'ipython = IPython.core.ipapi:launch_new_instance',
146 'ipython = IPython.core.ipapi:launch_new_instance',
147 'pycolor = IPython.PyColorize:main',
147 'pycolor = IPython.utils.PyColorize:main',
148 'ipcontroller = IPython.kernel.scripts.ipcontroller:main',
148 'ipcontroller = IPython.kernel.scripts.ipcontroller:main',
149 'ipengine = IPython.kernel.scripts.ipengine:main',
149 'ipengine = IPython.kernel.scripts.ipengine:main',
150 'ipcluster = IPython.kernel.scripts.ipcluster:main',
150 'ipcluster = IPython.kernel.scripts.ipcluster:main',
151 'ipythonx = IPython.frontend.wx.ipythonx:main',
151 'ipythonx = IPython.frontend.wx.ipythonx:main',
152 'iptest = IPython.testing.iptest:main',
152 'iptest = IPython.testing.iptest:main',
153 'irunner = IPython.lib.irunner:main'
153 ]
154 ]
154 }
155 }
155 setup_args['extras_require'] = dict(
156 setup_args['extras_require'] = dict(
156 kernel = [
157 kernel = [
157 'zope.interface>=3.4.1',
158 'zope.interface>=3.4.1',
158 'Twisted>=8.0.1',
159 'Twisted>=8.0.1',
159 'foolscap>=0.2.6'
160 'foolscap>=0.2.6'
160 ],
161 ],
161 doc='Sphinx>=0.3',
162 doc='Sphinx>=0.3',
162 test='nose>=0.10.1',
163 test='nose>=0.10.1',
163 security='pyOpenSSL>=0.6'
164 security='pyOpenSSL>=0.6'
164 )
165 )
165 # Allow setuptools to handle the scripts
166 # Allow setuptools to handle the scripts
166 scripts = []
167 scripts = []
167 else:
168 else:
168 # package_data of setuptools was introduced to distutils in 2.4
169 # package_data of setuptools was introduced to distutils in 2.4
169 cfgfiles = filter(isfile, glob('IPython/UserConfig/*'))
170 cfgfiles = filter(isfile, glob(pjoin('IPython','config','userconfig')))
170 if sys.version_info < (2,4):
171 if sys.version_info < (2,4):
171 data_files.append(('lib', 'IPython/UserConfig', cfgfiles))
172 data_files.append(('lib', pjoin('IPython','config','userconfig'), cfgfiles))
172 # If we are running without setuptools, call this function which will
173 # If we are running without setuptools, call this function which will
173 # check for dependencies an inform the user what is needed. This is
174 # check for dependencies an inform the user what is needed. This is
174 # just to make life easy for users.
175 # just to make life easy for users.
175 check_for_dependencies()
176 check_for_dependencies()
176
177
177
178
178 #---------------------------------------------------------------------------
179 #---------------------------------------------------------------------------
179 # Do the actual setup now
180 # Do the actual setup now
180 #---------------------------------------------------------------------------
181 #---------------------------------------------------------------------------
181
182
182 setup_args['packages'] = packages
183 setup_args['packages'] = packages
183 setup_args['package_data'] = package_data
184 setup_args['package_data'] = package_data
184 setup_args['scripts'] = scripts
185 setup_args['scripts'] = scripts
185 setup_args['data_files'] = data_files
186 setup_args['data_files'] = data_files
186 setup_args.update(setuptools_extra_args)
187 setup_args.update(setuptools_extra_args)
187
188
188 if __name__ == '__main__':
189 if __name__ == '__main__':
189 setup(**setup_args)
190 setup(**setup_args)
@@ -1,279 +1,296 b''
1 # encoding: utf-8
1 # encoding: utf-8
2
2
3 """
3 """
4 This module defines the things that are used in setup.py for building IPython
4 This module defines the things that are used in setup.py for building IPython
5
5
6 This includes:
6 This includes:
7
7
8 * The basic arguments to setup
8 * The basic arguments to setup
9 * Functions for finding things like packages, package data, etc.
9 * Functions for finding things like packages, package data, etc.
10 * A function for checking dependencies.
10 * A function for checking dependencies.
11 """
11 """
12
12
13 __docformat__ = "restructuredtext en"
13 __docformat__ = "restructuredtext en"
14
14
15 #-------------------------------------------------------------------------------
15 #-------------------------------------------------------------------------------
16 # Copyright (C) 2008 The IPython Development Team
16 # Copyright (C) 2008 The IPython Development Team
17 #
17 #
18 # Distributed under the terms of the BSD License. The full license is in
18 # Distributed under the terms of the BSD License. The full license is in
19 # the file COPYING, distributed as part of this software.
19 # the file COPYING, distributed as part of this software.
20 #-------------------------------------------------------------------------------
20 #-------------------------------------------------------------------------------
21
21
22 #-------------------------------------------------------------------------------
22 #-------------------------------------------------------------------------------
23 # Imports
23 # Imports
24 #-------------------------------------------------------------------------------
24 #-------------------------------------------------------------------------------
25
25
26 import os, sys
26 import os, sys
27
27
28 from glob import glob
28 from glob import glob
29
29
30 from setupext import install_data_ext
30 from setupext import install_data_ext
31
31
32 #-------------------------------------------------------------------------------
32 #-------------------------------------------------------------------------------
33 # Useful globals and utility functions
33 # Useful globals and utility functions
34 #-------------------------------------------------------------------------------
34 #-------------------------------------------------------------------------------
35
35
36 # A few handy globals
36 # A few handy globals
37 isfile = os.path.isfile
37 isfile = os.path.isfile
38 pjoin = os.path.join
38 pjoin = os.path.join
39
39
40 def oscmd(s):
40 def oscmd(s):
41 print ">", s
41 print ">", s
42 os.system(s)
42 os.system(s)
43
43
44 # A little utility we'll need below, since glob() does NOT allow you to do
44 # A little utility we'll need below, since glob() does NOT allow you to do
45 # exclusion on multiple endings!
45 # exclusion on multiple endings!
46 def file_doesnt_endwith(test,endings):
46 def file_doesnt_endwith(test,endings):
47 """Return true if test is a file and its name does NOT end with any
47 """Return true if test is a file and its name does NOT end with any
48 of the strings listed in endings."""
48 of the strings listed in endings."""
49 if not isfile(test):
49 if not isfile(test):
50 return False
50 return False
51 for e in endings:
51 for e in endings:
52 if test.endswith(e):
52 if test.endswith(e):
53 return False
53 return False
54 return True
54 return True
55
55
56 #---------------------------------------------------------------------------
56 #---------------------------------------------------------------------------
57 # Basic project information
57 # Basic project information
58 #---------------------------------------------------------------------------
58 #---------------------------------------------------------------------------
59
59
60 # Release.py contains version, authors, license, url, keywords, etc.
60 # Release.py contains version, authors, license, url, keywords, etc.
61 execfile(pjoin('IPython','Release.py'))
61 execfile(pjoin('IPython','core','release.py'))
62
62
63 # Create a dict with the basic information
63 # Create a dict with the basic information
64 # This dict is eventually passed to setup after additional keys are added.
64 # This dict is eventually passed to setup after additional keys are added.
65 setup_args = dict(
65 setup_args = dict(
66 name = name,
66 name = name,
67 version = version,
67 version = version,
68 description = description,
68 description = description,
69 long_description = long_description,
69 long_description = long_description,
70 author = author,
70 author = author,
71 author_email = author_email,
71 author_email = author_email,
72 url = url,
72 url = url,
73 download_url = download_url,
73 download_url = download_url,
74 license = license,
74 license = license,
75 platforms = platforms,
75 platforms = platforms,
76 keywords = keywords,
76 keywords = keywords,
77 cmdclass = {'install_data': install_data_ext},
77 cmdclass = {'install_data': install_data_ext},
78 )
78 )
79
79
80
80
81 #---------------------------------------------------------------------------
81 #---------------------------------------------------------------------------
82 # Find packages
82 # Find packages
83 #---------------------------------------------------------------------------
83 #---------------------------------------------------------------------------
84
84
85 def add_package(packages,pname,config=False,tests=False,scripts=False,
85 def add_package(packages,pname,config=False,tests=False,scripts=False,
86 others=None):
86 others=None):
87 """
87 """
88 Add a package to the list of packages, including certain subpackages.
88 Add a package to the list of packages, including certain subpackages.
89 """
89 """
90 packages.append('.'.join(['IPython',pname]))
90 packages.append('.'.join(['IPython',pname]))
91 if config:
91 if config:
92 packages.append('.'.join(['IPython',pname,'config']))
92 packages.append('.'.join(['IPython',pname,'config']))
93 if tests:
93 if tests:
94 packages.append('.'.join(['IPython',pname,'tests']))
94 packages.append('.'.join(['IPython',pname,'tests']))
95 if scripts:
95 if scripts:
96 packages.append('.'.join(['IPython',pname,'scripts']))
96 packages.append('.'.join(['IPython',pname,'scripts']))
97 if others is not None:
97 if others is not None:
98 for o in others:
98 for o in others:
99 packages.append('.'.join(['IPython',pname,o]))
99 packages.append('.'.join(['IPython',pname,o]))
100
100
101 def find_packages():
101 def find_packages():
102 """
102 """
103 Find all of IPython's packages.
103 Find all of IPython's packages.
104 """
104 """
105 packages = ['IPython']
105 packages = ['IPython']
106 add_package(packages, 'config', tests=True)
106 add_package(packages, 'config', tests=True)
107 add_package(packages, 'config.userconfig')
108 add_package(packages, 'core', tests=True)
109 add_package(packages, 'deathrow', tests=True)
107 add_package(packages , 'Extensions')
110 add_package(packages , 'Extensions')
108 add_package(packages, 'external')
111 add_package(packages, 'external')
109 add_package(packages, 'gui')
110 add_package(packages, 'gui.wx')
111 add_package(packages, 'frontend', tests=True)
112 add_package(packages, 'frontend', tests=True)
113 # Don't include the cocoa frontend for now as it is not stable
114 if sys.platform == 'darwin' and False:
115 add_package(packages, 'frontend.cocoa', tests=True, others=['plugin'])
116 add_package(packages, 'frontend.cocoa.examples')
117 add_package(packages, 'frontend.cocoa.examples.IPython1Sandbox')
118 add_package(packages, 'frontend.cocoa.examples.IPython1Sandbox.English.lproj')
112 add_package(packages, 'frontend.process')
119 add_package(packages, 'frontend.process')
113 add_package(packages, 'frontend.wx')
120 add_package(packages, 'frontend.wx')
114 add_package(packages, 'frontend.cocoa', tests=True)
121 add_package(packages, 'gui')
122 add_package(packages, 'gui.wx')
115 add_package(packages, 'kernel', config=True, tests=True, scripts=True)
123 add_package(packages, 'kernel', config=True, tests=True, scripts=True)
116 add_package(packages, 'kernel.core', config=True, tests=True)
124 add_package(packages, 'kernel.core', config=True, tests=True)
125 add_package(packages, 'lib', tests=True)
126 add_package(packages, 'quarantine', tests=True)
127 add_package(packages, 'scripts')
117 add_package(packages, 'testing', tests=True)
128 add_package(packages, 'testing', tests=True)
118 add_package(packages, 'tests')
119 add_package(packages, 'testing.plugin', tests=False)
129 add_package(packages, 'testing.plugin', tests=False)
120 add_package(packages, 'tools', tests=True)
130 add_package(packages, 'utils', tests=True)
121 add_package(packages, 'UserConfig')
122 return packages
131 return packages
123
132
124 #---------------------------------------------------------------------------
133 #---------------------------------------------------------------------------
125 # Find package data
134 # Find package data
126 #---------------------------------------------------------------------------
135 #---------------------------------------------------------------------------
127
136
128 def find_package_data():
137 def find_package_data():
129 """
138 """
130 Find IPython's package_data.
139 Find IPython's package_data.
131 """
140 """
132 # This is not enough for these things to appear in an sdist.
141 # This is not enough for these things to appear in an sdist.
133 # We need to muck with the MANIFEST to get this to work
142 # We need to muck with the MANIFEST to get this to work
134 package_data = {
143 package_data = {
135 'IPython.UserConfig' : ['*'],
144 'IPython.config.userconfig' : ['*'],
136 'IPython.tools.tests' : ['*.txt'],
137 'IPython.testing' : ['*.txt']
145 'IPython.testing' : ['*.txt']
138 }
146 }
139 return package_data
147 return package_data
140
148
141
149
142 #---------------------------------------------------------------------------
150 #---------------------------------------------------------------------------
143 # Find data files
151 # Find data files
144 #---------------------------------------------------------------------------
152 #---------------------------------------------------------------------------
145
153
146 def make_dir_struct(tag,base,out_base):
154 def make_dir_struct(tag,base,out_base):
147 """Make the directory structure of all files below a starting dir.
155 """Make the directory structure of all files below a starting dir.
148
156
149 This is just a convenience routine to help build a nested directory
157 This is just a convenience routine to help build a nested directory
150 hierarchy because distutils is too stupid to do this by itself.
158 hierarchy because distutils is too stupid to do this by itself.
151
159
152 XXX - this needs a proper docstring!
160 XXX - this needs a proper docstring!
153 """
161 """
154
162
155 # we'll use these a lot below
163 # we'll use these a lot below
156 lbase = len(base)
164 lbase = len(base)
157 pathsep = os.path.sep
165 pathsep = os.path.sep
158 lpathsep = len(pathsep)
166 lpathsep = len(pathsep)
159
167
160 out = []
168 out = []
161 for (dirpath,dirnames,filenames) in os.walk(base):
169 for (dirpath,dirnames,filenames) in os.walk(base):
162 # we need to strip out the dirpath from the base to map it to the
170 # we need to strip out the dirpath from the base to map it to the
163 # output (installation) path. This requires possibly stripping the
171 # output (installation) path. This requires possibly stripping the
164 # path separator, because otherwise pjoin will not work correctly
172 # path separator, because otherwise pjoin will not work correctly
165 # (pjoin('foo/','/bar') returns '/bar').
173 # (pjoin('foo/','/bar') returns '/bar').
166
174
167 dp_eff = dirpath[lbase:]
175 dp_eff = dirpath[lbase:]
168 if dp_eff.startswith(pathsep):
176 if dp_eff.startswith(pathsep):
169 dp_eff = dp_eff[lpathsep:]
177 dp_eff = dp_eff[lpathsep:]
170 # The output path must be anchored at the out_base marker
178 # The output path must be anchored at the out_base marker
171 out_path = pjoin(out_base,dp_eff)
179 out_path = pjoin(out_base,dp_eff)
172 # Now we can generate the final filenames. Since os.walk only produces
180 # Now we can generate the final filenames. Since os.walk only produces
173 # filenames, we must join back with the dirpath to get full valid file
181 # filenames, we must join back with the dirpath to get full valid file
174 # paths:
182 # paths:
175 pfiles = [pjoin(dirpath,f) for f in filenames]
183 pfiles = [pjoin(dirpath,f) for f in filenames]
176 # Finally, generate the entry we need, which is a triple of (tag,output
184 # Finally, generate the entry we need, which is a triple of (tag,output
177 # path, files) for use as a data_files parameter in install_data.
185 # path, files) for use as a data_files parameter in install_data.
178 out.append((tag,out_path,pfiles))
186 out.append((tag,out_path,pfiles))
179
187
180 return out
188 return out
181
189
182
190
183 def find_data_files():
191 def find_data_files():
184 """
192 """
185 Find IPython's data_files.
193 Find IPython's data_files.
186
194
187 Most of these are docs.
195 Most of these are docs.
188 """
196 """
189
197
190 docdirbase = 'share/doc/ipython'
198 docdirbase = pjoin('share', 'doc', 'ipython')
191 manpagebase = 'share/man/man1'
199 manpagebase = pjoin('share', 'man', 'man1')
192
200
193 # Simple file lists can be made by hand
201 # Simple file lists can be made by hand
194 manpages = filter(isfile, glob('docs/man/*.1.gz'))
202 manpages = filter(isfile, glob(pjoin('docs','man','*.1.gz')))
195 igridhelpfiles = filter(isfile, glob('IPython/Extensions/igrid_help.*'))
203 igridhelpfiles = filter(isfile, glob(pjoin('IPython','Extensions','igrid_help.*')))
196
204
197 # For nested structures, use the utility above
205 # For nested structures, use the utility above
198 example_files = make_dir_struct('data','docs/examples',
206 example_files = make_dir_struct(
199 pjoin(docdirbase,'examples'))
207 'data',
200 manual_files = make_dir_struct('data','docs/dist',pjoin(docdirbase,'manual'))
208 pjoin('docs','examples'),
209 pjoin(docdirbase,'examples')
210 )
211 manual_files = make_dir_struct(
212 'data',
213 pjoin('docs','dist'),
214 pjoin(docdirbase,'manual')
215 )
201
216
202 # And assemble the entire output list
217 # And assemble the entire output list
203 data_files = [ ('data',manpagebase, manpages),
218 data_files = [ ('data',manpagebase, manpages),
204 ('data',pjoin(docdirbase,'extensions'),igridhelpfiles),
219 ('data',pjoin(docdirbase,'extensions'),igridhelpfiles),
205 ] + manual_files + example_files
220 ] + manual_files + example_files
206
221
207 ## import pprint # dbg
222 ## import pprint # dbg
208 ## print '*'*80
223 ## print '*'*80
209 ## print 'data files'
224 ## print 'data files'
210 ## pprint.pprint(data_files)
225 ## pprint.pprint(data_files)
211 ## print '*'*80
226 ## print '*'*80
212
227
213 return data_files
228 return data_files
214
229
215 #---------------------------------------------------------------------------
230 #---------------------------------------------------------------------------
216 # Find scripts
231 # Find scripts
217 #---------------------------------------------------------------------------
232 #---------------------------------------------------------------------------
218
233
219 def find_scripts():
234 def find_scripts():
220 """
235 """
221 Find IPython's scripts.
236 Find IPython's scripts.
222 """
237 """
223 scripts = ['IPython/kernel/scripts/ipengine',
238 kernel_scripts = pjoin('IPython','kernel','scripts')
224 'IPython/kernel/scripts/ipcontroller',
239 main_scripts = pjoin('IPython','scripts')
225 'IPython/kernel/scripts/ipcluster',
240 scripts = [pjoin(kernel_scripts, 'ipengine'),
226 'scripts/ipython',
241 pjoin(kernel_scripts, 'ipcontroller'),
227 'scripts/ipythonx',
242 pjoin(kernel_scripts, 'ipcluster'),
228 'scripts/ipython-wx',
243 pjoin(main_scripts, 'ipython'),
229 'scripts/pycolor',
244 pjoin(main_scripts, 'ipythonx'),
230 'scripts/irunner',
245 pjoin(main_scripts, 'ipython-wx'),
231 'scripts/iptest',
246 pjoin(main_scripts, 'pycolor'),
247 pjoin(main_scripts, 'irunner'),
248 pjoin(main_scripts, 'iptest')
232 ]
249 ]
233
250
234 # Script to be run by the windows binary installer after the default setup
251 # Script to be run by the windows binary installer after the default setup
235 # routine, to add shortcuts and similar windows-only things. Windows
252 # routine, to add shortcuts and similar windows-only things. Windows
236 # post-install scripts MUST reside in the scripts/ dir, otherwise distutils
253 # post-install scripts MUST reside in the scripts/ dir, otherwise distutils
237 # doesn't find them.
254 # doesn't find them.
238 if 'bdist_wininst' in sys.argv:
255 if 'bdist_wininst' in sys.argv:
239 if len(sys.argv) > 2 and ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
256 if len(sys.argv) > 2 and ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
240 print >> sys.stderr,"ERROR: bdist_wininst must be run alone. Exiting."
257 print >> sys.stderr,"ERROR: bdist_wininst must be run alone. Exiting."
241 sys.exit(1)
258 sys.exit(1)
242 scripts.append('scripts/ipython_win_post_install.py')
259 scripts.append(pjoin(main_scripts,'ipython_win_post_install.py'))
243
260
244 return scripts
261 return scripts
245
262
246 #---------------------------------------------------------------------------
263 #---------------------------------------------------------------------------
247 # Verify all dependencies
264 # Verify all dependencies
248 #---------------------------------------------------------------------------
265 #---------------------------------------------------------------------------
249
266
250 def check_for_dependencies():
267 def check_for_dependencies():
251 """Check for IPython's dependencies.
268 """Check for IPython's dependencies.
252
269
253 This function should NOT be called if running under setuptools!
270 This function should NOT be called if running under setuptools!
254 """
271 """
255 from setupext.setupext import (
272 from setupext.setupext import (
256 print_line, print_raw, print_status, print_message,
273 print_line, print_raw, print_status, print_message,
257 check_for_zopeinterface, check_for_twisted,
274 check_for_zopeinterface, check_for_twisted,
258 check_for_foolscap, check_for_pyopenssl,
275 check_for_foolscap, check_for_pyopenssl,
259 check_for_sphinx, check_for_pygments,
276 check_for_sphinx, check_for_pygments,
260 check_for_nose, check_for_pexpect
277 check_for_nose, check_for_pexpect
261 )
278 )
262 print_line()
279 print_line()
263 print_raw("BUILDING IPYTHON")
280 print_raw("BUILDING IPYTHON")
264 print_status('python', sys.version)
281 print_status('python', sys.version)
265 print_status('platform', sys.platform)
282 print_status('platform', sys.platform)
266 if sys.platform == 'win32':
283 if sys.platform == 'win32':
267 print_status('Windows version', sys.getwindowsversion())
284 print_status('Windows version', sys.getwindowsversion())
268
285
269 print_raw("")
286 print_raw("")
270 print_raw("OPTIONAL DEPENDENCIES")
287 print_raw("OPTIONAL DEPENDENCIES")
271
288
272 check_for_zopeinterface()
289 check_for_zopeinterface()
273 check_for_twisted()
290 check_for_twisted()
274 check_for_foolscap()
291 check_for_foolscap()
275 check_for_pyopenssl()
292 check_for_pyopenssl()
276 check_for_sphinx()
293 check_for_sphinx()
277 check_for_pygments()
294 check_for_pygments()
278 check_for_nose()
295 check_for_nose()
279 check_for_pexpect()
296 check_for_pexpect()
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now