Show More
@@ -1,491 +1,497 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The IPython controller application. |
|
4 | The IPython controller application. | |
5 |
|
5 | |||
6 | Authors: |
|
6 | Authors: | |
7 |
|
7 | |||
8 | * Brian Granger |
|
8 | * Brian Granger | |
9 | * MinRK |
|
9 | * MinRK | |
10 |
|
10 | |||
11 | """ |
|
11 | """ | |
12 |
|
12 | |||
13 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
14 | # Copyright (C) 2008-2011 The IPython Development Team |
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |
15 | # |
|
15 | # | |
16 | # Distributed under the terms of the BSD License. The full license is in |
|
16 | # Distributed under the terms of the BSD License. The full license is in | |
17 | # the file COPYING, distributed as part of this software. |
|
17 | # the file COPYING, distributed as part of this software. | |
18 | #----------------------------------------------------------------------------- |
|
18 | #----------------------------------------------------------------------------- | |
19 |
|
19 | |||
20 | #----------------------------------------------------------------------------- |
|
20 | #----------------------------------------------------------------------------- | |
21 | # Imports |
|
21 | # Imports | |
22 | #----------------------------------------------------------------------------- |
|
22 | #----------------------------------------------------------------------------- | |
23 |
|
23 | |||
24 | from __future__ import with_statement |
|
24 | from __future__ import with_statement | |
25 |
|
25 | |||
26 | import json |
|
26 | import json | |
27 | import os |
|
27 | import os | |
28 | import socket |
|
28 | import socket | |
29 | import stat |
|
29 | import stat | |
30 | import sys |
|
30 | import sys | |
31 |
|
31 | |||
32 | from multiprocessing import Process |
|
32 | from multiprocessing import Process | |
33 | from signal import signal, SIGINT, SIGABRT, SIGTERM |
|
33 | from signal import signal, SIGINT, SIGABRT, SIGTERM | |
34 |
|
34 | |||
35 | import zmq |
|
35 | import zmq | |
36 | from zmq.devices import ProcessMonitoredQueue |
|
36 | from zmq.devices import ProcessMonitoredQueue | |
37 | from zmq.log.handlers import PUBHandler |
|
37 | from zmq.log.handlers import PUBHandler | |
38 |
|
38 | |||
39 | from IPython.core.profiledir import ProfileDir |
|
39 | from IPython.core.profiledir import ProfileDir | |
40 |
|
40 | |||
41 | from IPython.parallel.apps.baseapp import ( |
|
41 | from IPython.parallel.apps.baseapp import ( | |
42 | BaseParallelApplication, |
|
42 | BaseParallelApplication, | |
43 | base_aliases, |
|
43 | base_aliases, | |
44 | base_flags, |
|
44 | base_flags, | |
45 | catch_config_error, |
|
45 | catch_config_error, | |
46 | ) |
|
46 | ) | |
47 | from IPython.utils.importstring import import_item |
|
47 | from IPython.utils.importstring import import_item | |
48 | from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError |
|
48 | from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError | |
49 |
|
49 | |||
50 | from IPython.zmq.session import ( |
|
50 | from IPython.zmq.session import ( | |
51 | Session, session_aliases, session_flags, default_secure |
|
51 | Session, session_aliases, session_flags, default_secure | |
52 | ) |
|
52 | ) | |
53 |
|
53 | |||
54 | from IPython.parallel.controller.heartmonitor import HeartMonitor |
|
54 | from IPython.parallel.controller.heartmonitor import HeartMonitor | |
55 | from IPython.parallel.controller.hub import HubFactory |
|
55 | from IPython.parallel.controller.hub import HubFactory | |
56 | from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler |
|
56 | from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler | |
57 | from IPython.parallel.controller.sqlitedb import SQLiteDB |
|
57 | from IPython.parallel.controller.sqlitedb import SQLiteDB | |
58 |
|
58 | |||
59 | from IPython.parallel.util import split_url, disambiguate_url |
|
59 | from IPython.parallel.util import split_url, disambiguate_url | |
60 |
|
60 | |||
61 | # conditional import of MongoDB backend class |
|
61 | # conditional import of MongoDB backend class | |
62 |
|
62 | |||
63 | try: |
|
63 | try: | |
64 | from IPython.parallel.controller.mongodb import MongoDB |
|
64 | from IPython.parallel.controller.mongodb import MongoDB | |
65 | except ImportError: |
|
65 | except ImportError: | |
66 | maybe_mongo = [] |
|
66 | maybe_mongo = [] | |
67 | else: |
|
67 | else: | |
68 | maybe_mongo = [MongoDB] |
|
68 | maybe_mongo = [MongoDB] | |
69 |
|
69 | |||
70 |
|
70 | |||
71 | #----------------------------------------------------------------------------- |
|
71 | #----------------------------------------------------------------------------- | |
72 | # Module level variables |
|
72 | # Module level variables | |
73 | #----------------------------------------------------------------------------- |
|
73 | #----------------------------------------------------------------------------- | |
74 |
|
74 | |||
75 |
|
75 | |||
76 | #: The default config file name for this application |
|
76 | #: The default config file name for this application | |
77 | default_config_file_name = u'ipcontroller_config.py' |
|
77 | default_config_file_name = u'ipcontroller_config.py' | |
78 |
|
78 | |||
79 |
|
79 | |||
80 | _description = """Start the IPython controller for parallel computing. |
|
80 | _description = """Start the IPython controller for parallel computing. | |
81 |
|
81 | |||
82 | The IPython controller provides a gateway between the IPython engines and |
|
82 | The IPython controller provides a gateway between the IPython engines and | |
83 | clients. The controller needs to be started before the engines and can be |
|
83 | clients. The controller needs to be started before the engines and can be | |
84 | configured using command line options or using a cluster directory. Cluster |
|
84 | configured using command line options or using a cluster directory. Cluster | |
85 | directories contain config, log and security files and are usually located in |
|
85 | directories contain config, log and security files and are usually located in | |
86 | your ipython directory and named as "profile_name". See the `profile` |
|
86 | your ipython directory and named as "profile_name". See the `profile` | |
87 | and `profile-dir` options for details. |
|
87 | and `profile-dir` options for details. | |
88 | """ |
|
88 | """ | |
89 |
|
89 | |||
90 | _examples = """ |
|
90 | _examples = """ | |
91 | ipcontroller --ip=192.168.0.1 --port=1000 # listen on ip, port for engines |
|
91 | ipcontroller --ip=192.168.0.1 --port=1000 # listen on ip, port for engines | |
92 | ipcontroller --scheme=pure # use the pure zeromq scheduler |
|
92 | ipcontroller --scheme=pure # use the pure zeromq scheduler | |
93 | """ |
|
93 | """ | |
94 |
|
94 | |||
95 |
|
95 | |||
96 | #----------------------------------------------------------------------------- |
|
96 | #----------------------------------------------------------------------------- | |
97 | # The main application |
|
97 | # The main application | |
98 | #----------------------------------------------------------------------------- |
|
98 | #----------------------------------------------------------------------------- | |
99 | flags = {} |
|
99 | flags = {} | |
100 | flags.update(base_flags) |
|
100 | flags.update(base_flags) | |
101 | flags.update({ |
|
101 | flags.update({ | |
102 | 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}}, |
|
102 | 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}}, | |
103 | 'Use threads instead of processes for the schedulers'), |
|
103 | 'Use threads instead of processes for the schedulers'), | |
104 | 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}}, |
|
104 | 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}}, | |
105 | 'use the SQLiteDB backend'), |
|
105 | 'use the SQLiteDB backend'), | |
106 | 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}}, |
|
106 | 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}}, | |
107 | 'use the MongoDB backend'), |
|
107 | 'use the MongoDB backend'), | |
108 | 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}}, |
|
108 | 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}}, | |
109 | 'use the in-memory DictDB backend'), |
|
109 | 'use the in-memory DictDB backend'), | |
110 | 'nodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.NoDB'}}, |
|
110 | 'nodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.NoDB'}}, | |
111 | """use dummy DB backend, which doesn't store any information. |
|
111 | """use dummy DB backend, which doesn't store any information. | |
112 |
|
112 | |||
113 | This is the default as of IPython 0.13. |
|
113 | This is the default as of IPython 0.13. | |
114 |
|
114 | |||
115 | To enable delayed or repeated retrieval of results from the Hub, |
|
115 | To enable delayed or repeated retrieval of results from the Hub, | |
116 | select one of the true db backends. |
|
116 | select one of the true db backends. | |
117 | """), |
|
117 | """), | |
118 | 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}}, |
|
118 | 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}}, | |
119 | 'reuse existing json connection files') |
|
119 | 'reuse existing json connection files') | |
120 | }) |
|
120 | }) | |
121 |
|
121 | |||
122 | flags.update(session_flags) |
|
122 | flags.update(session_flags) | |
123 |
|
123 | |||
124 | aliases = dict( |
|
124 | aliases = dict( | |
125 | ssh = 'IPControllerApp.ssh_server', |
|
125 | ssh = 'IPControllerApp.ssh_server', | |
126 | enginessh = 'IPControllerApp.engine_ssh_server', |
|
126 | enginessh = 'IPControllerApp.engine_ssh_server', | |
127 | location = 'IPControllerApp.location', |
|
127 | location = 'IPControllerApp.location', | |
128 |
|
128 | |||
129 | url = 'HubFactory.url', |
|
129 | url = 'HubFactory.url', | |
130 | ip = 'HubFactory.ip', |
|
130 | ip = 'HubFactory.ip', | |
131 | transport = 'HubFactory.transport', |
|
131 | transport = 'HubFactory.transport', | |
132 | port = 'HubFactory.regport', |
|
132 | port = 'HubFactory.regport', | |
133 |
|
133 | |||
134 | ping = 'HeartMonitor.period', |
|
134 | ping = 'HeartMonitor.period', | |
135 |
|
135 | |||
136 | scheme = 'TaskScheduler.scheme_name', |
|
136 | scheme = 'TaskScheduler.scheme_name', | |
137 | hwm = 'TaskScheduler.hwm', |
|
137 | hwm = 'TaskScheduler.hwm', | |
138 | ) |
|
138 | ) | |
139 | aliases.update(base_aliases) |
|
139 | aliases.update(base_aliases) | |
140 | aliases.update(session_aliases) |
|
140 | aliases.update(session_aliases) | |
141 |
|
141 | |||
142 | class IPControllerApp(BaseParallelApplication): |
|
142 | class IPControllerApp(BaseParallelApplication): | |
143 |
|
143 | |||
144 | name = u'ipcontroller' |
|
144 | name = u'ipcontroller' | |
145 | description = _description |
|
145 | description = _description | |
146 | examples = _examples |
|
146 | examples = _examples | |
147 | config_file_name = Unicode(default_config_file_name) |
|
147 | config_file_name = Unicode(default_config_file_name) | |
148 | classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo |
|
148 | classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo | |
149 |
|
149 | |||
150 | # change default to True |
|
150 | # change default to True | |
151 | auto_create = Bool(True, config=True, |
|
151 | auto_create = Bool(True, config=True, | |
152 | help="""Whether to create profile dir if it doesn't exist.""") |
|
152 | help="""Whether to create profile dir if it doesn't exist.""") | |
153 |
|
153 | |||
154 | reuse_files = Bool(False, config=True, |
|
154 | reuse_files = Bool(False, config=True, | |
155 | help="""Whether to reuse existing json connection files. |
|
155 | help="""Whether to reuse existing json connection files. | |
156 | If False, connection files will be removed on a clean exit. |
|
156 | If False, connection files will be removed on a clean exit. | |
157 | """ |
|
157 | """ | |
158 | ) |
|
158 | ) | |
159 | ssh_server = Unicode(u'', config=True, |
|
159 | ssh_server = Unicode(u'', config=True, | |
160 | help="""ssh url for clients to use when connecting to the Controller |
|
160 | help="""ssh url for clients to use when connecting to the Controller | |
161 | processes. It should be of the form: [user@]server[:port]. The |
|
161 | processes. It should be of the form: [user@]server[:port]. The | |
162 | Controller's listening addresses must be accessible from the ssh server""", |
|
162 | Controller's listening addresses must be accessible from the ssh server""", | |
163 | ) |
|
163 | ) | |
164 | engine_ssh_server = Unicode(u'', config=True, |
|
164 | engine_ssh_server = Unicode(u'', config=True, | |
165 | help="""ssh url for engines to use when connecting to the Controller |
|
165 | help="""ssh url for engines to use when connecting to the Controller | |
166 | processes. It should be of the form: [user@]server[:port]. The |
|
166 | processes. It should be of the form: [user@]server[:port]. The | |
167 | Controller's listening addresses must be accessible from the ssh server""", |
|
167 | Controller's listening addresses must be accessible from the ssh server""", | |
168 | ) |
|
168 | ) | |
169 | location = Unicode(u'', config=True, |
|
169 | location = Unicode(u'', config=True, | |
170 | help="""The external IP or domain name of the Controller, used for disambiguating |
|
170 | help="""The external IP or domain name of the Controller, used for disambiguating | |
171 | engine and client connections.""", |
|
171 | engine and client connections.""", | |
172 | ) |
|
172 | ) | |
173 | import_statements = List([], config=True, |
|
173 | import_statements = List([], config=True, | |
174 | help="import statements to be run at startup. Necessary in some environments" |
|
174 | help="import statements to be run at startup. Necessary in some environments" | |
175 | ) |
|
175 | ) | |
176 |
|
176 | |||
177 | use_threads = Bool(False, config=True, |
|
177 | use_threads = Bool(False, config=True, | |
178 | help='Use threads instead of processes for the schedulers', |
|
178 | help='Use threads instead of processes for the schedulers', | |
179 | ) |
|
179 | ) | |
180 |
|
180 | |||
181 | engine_json_file = Unicode('ipcontroller-engine.json', config=True, |
|
181 | engine_json_file = Unicode('ipcontroller-engine.json', config=True, | |
182 | help="JSON filename where engine connection info will be stored.") |
|
182 | help="JSON filename where engine connection info will be stored.") | |
183 | client_json_file = Unicode('ipcontroller-client.json', config=True, |
|
183 | client_json_file = Unicode('ipcontroller-client.json', config=True, | |
184 | help="JSON filename where client connection info will be stored.") |
|
184 | help="JSON filename where client connection info will be stored.") | |
185 |
|
185 | |||
186 | def _cluster_id_changed(self, name, old, new): |
|
186 | def _cluster_id_changed(self, name, old, new): | |
187 | super(IPControllerApp, self)._cluster_id_changed(name, old, new) |
|
187 | super(IPControllerApp, self)._cluster_id_changed(name, old, new) | |
188 | self.engine_json_file = "%s-engine.json" % self.name |
|
188 | self.engine_json_file = "%s-engine.json" % self.name | |
189 | self.client_json_file = "%s-client.json" % self.name |
|
189 | self.client_json_file = "%s-client.json" % self.name | |
190 |
|
190 | |||
191 |
|
191 | |||
192 | # internal |
|
192 | # internal | |
193 | children = List() |
|
193 | children = List() | |
194 | mq_class = Unicode('zmq.devices.ProcessMonitoredQueue') |
|
194 | mq_class = Unicode('zmq.devices.ProcessMonitoredQueue') | |
195 |
|
195 | |||
196 | def _use_threads_changed(self, name, old, new): |
|
196 | def _use_threads_changed(self, name, old, new): | |
197 | self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process') |
|
197 | self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process') | |
198 |
|
198 | |||
199 | write_connection_files = Bool(True, |
|
199 | write_connection_files = Bool(True, | |
200 | help="""Whether to write connection files to disk. |
|
200 | help="""Whether to write connection files to disk. | |
201 | True in all cases other than runs with `reuse_files=True` *after the first* |
|
201 | True in all cases other than runs with `reuse_files=True` *after the first* | |
202 | """ |
|
202 | """ | |
203 | ) |
|
203 | ) | |
204 |
|
204 | |||
205 | aliases = Dict(aliases) |
|
205 | aliases = Dict(aliases) | |
206 | flags = Dict(flags) |
|
206 | flags = Dict(flags) | |
207 |
|
207 | |||
208 |
|
208 | |||
209 | def save_connection_dict(self, fname, cdict): |
|
209 | def save_connection_dict(self, fname, cdict): | |
210 | """save a connection dict to json file.""" |
|
210 | """save a connection dict to json file.""" | |
211 | c = self.config |
|
211 | c = self.config | |
212 |
url = cdict[' |
|
212 | url = cdict['registration'] | |
213 | location = cdict['location'] |
|
213 | location = cdict['location'] | |
214 | if not location: |
|
214 | if not location: | |
215 | try: |
|
215 | try: | |
216 | proto,ip,port = split_url(url) |
|
216 | proto,ip,port = split_url(url) | |
217 | except AssertionError: |
|
217 | except AssertionError: | |
218 | pass |
|
218 | pass | |
219 | else: |
|
219 | else: | |
220 | try: |
|
220 | try: | |
221 | location = socket.gethostbyname_ex(socket.gethostname())[2][-1] |
|
221 | location = socket.gethostbyname_ex(socket.gethostname())[2][-1] | |
222 | except (socket.gaierror, IndexError): |
|
222 | except (socket.gaierror, IndexError): | |
223 | self.log.warn("Could not identify this machine's IP, assuming 127.0.0.1." |
|
223 | self.log.warn("Could not identify this machine's IP, assuming 127.0.0.1." | |
224 | " You may need to specify '--location=<external_ip_address>' to help" |
|
224 | " You may need to specify '--location=<external_ip_address>' to help" | |
225 | " IPython decide when to connect via loopback.") |
|
225 | " IPython decide when to connect via loopback.") | |
226 | location = '127.0.0.1' |
|
226 | location = '127.0.0.1' | |
227 | cdict['location'] = location |
|
227 | cdict['location'] = location | |
228 | fname = os.path.join(self.profile_dir.security_dir, fname) |
|
228 | fname = os.path.join(self.profile_dir.security_dir, fname) | |
229 | self.log.info("writing connection info to %s", fname) |
|
229 | self.log.info("writing connection info to %s", fname) | |
230 | with open(fname, 'w') as f: |
|
230 | with open(fname, 'w') as f: | |
231 | f.write(json.dumps(cdict, indent=2)) |
|
231 | f.write(json.dumps(cdict, indent=2)) | |
232 | os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR) |
|
232 | os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR) | |
233 |
|
233 | |||
234 | def load_config_from_json(self): |
|
234 | def load_config_from_json(self): | |
235 | """load config from existing json connector files.""" |
|
235 | """load config from existing json connector files.""" | |
236 | c = self.config |
|
236 | c = self.config | |
237 | self.log.debug("loading config from JSON") |
|
237 | self.log.debug("loading config from JSON") | |
238 | # load from engine config |
|
238 | # load from engine config | |
239 | fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file) |
|
239 | fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file) | |
240 | self.log.info("loading connection info from %s", fname) |
|
240 | self.log.info("loading connection info from %s", fname) | |
241 | with open(fname) as f: |
|
241 | with open(fname) as f: | |
242 | cfg = json.loads(f.read()) |
|
242 | cfg = json.loads(f.read()) | |
243 | key = cfg['exec_key'] |
|
243 | key = cfg['exec_key'] | |
244 | # json gives unicode, Session.key wants bytes |
|
244 | # json gives unicode, Session.key wants bytes | |
245 | c.Session.key = key.encode('ascii') |
|
245 | c.Session.key = key.encode('ascii') | |
246 | xport,addr = cfg['url'].split('://') |
|
246 | xport,addr = cfg['url'].split('://') | |
247 | c.HubFactory.engine_transport = xport |
|
247 | c.HubFactory.engine_transport = xport | |
248 | ip,ports = addr.split(':') |
|
248 | ip,ports = addr.split(':') | |
249 | c.HubFactory.engine_ip = ip |
|
249 | c.HubFactory.engine_ip = ip | |
250 | c.HubFactory.regport = int(ports) |
|
250 | c.HubFactory.regport = int(ports) | |
251 | self.location = cfg['location'] |
|
251 | self.location = cfg['location'] | |
252 | if not self.engine_ssh_server: |
|
252 | if not self.engine_ssh_server: | |
253 | self.engine_ssh_server = cfg['ssh'] |
|
253 | self.engine_ssh_server = cfg['ssh'] | |
254 | # load client config |
|
254 | # load client config | |
255 | fname = os.path.join(self.profile_dir.security_dir, self.client_json_file) |
|
255 | fname = os.path.join(self.profile_dir.security_dir, self.client_json_file) | |
256 | self.log.info("loading connection info from %s", fname) |
|
256 | self.log.info("loading connection info from %s", fname) | |
257 | with open(fname) as f: |
|
257 | with open(fname) as f: | |
258 | cfg = json.loads(f.read()) |
|
258 | cfg = json.loads(f.read()) | |
259 | assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys" |
|
259 | assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys" | |
260 | xport,addr = cfg['url'].split('://') |
|
260 | xport,addr = cfg['url'].split('://') | |
261 | c.HubFactory.client_transport = xport |
|
261 | c.HubFactory.client_transport = xport | |
262 | ip,ports = addr.split(':') |
|
262 | ip,ports = addr.split(':') | |
263 | c.HubFactory.client_ip = ip |
|
263 | c.HubFactory.client_ip = ip | |
264 | if not self.ssh_server: |
|
264 | if not self.ssh_server: | |
265 | self.ssh_server = cfg['ssh'] |
|
265 | self.ssh_server = cfg['ssh'] | |
266 | assert int(ports) == c.HubFactory.regport, "regport mismatch" |
|
266 | assert int(ports) == c.HubFactory.regport, "regport mismatch" | |
267 |
|
267 | |||
268 | def cleanup_connection_files(self): |
|
268 | def cleanup_connection_files(self): | |
269 | if self.reuse_files: |
|
269 | if self.reuse_files: | |
270 | self.log.debug("leaving JSON connection files for reuse") |
|
270 | self.log.debug("leaving JSON connection files for reuse") | |
271 | return |
|
271 | return | |
272 | self.log.debug("cleaning up JSON connection files") |
|
272 | self.log.debug("cleaning up JSON connection files") | |
273 | for f in (self.client_json_file, self.engine_json_file): |
|
273 | for f in (self.client_json_file, self.engine_json_file): | |
274 | f = os.path.join(self.profile_dir.security_dir, f) |
|
274 | f = os.path.join(self.profile_dir.security_dir, f) | |
275 | try: |
|
275 | try: | |
276 | os.remove(f) |
|
276 | os.remove(f) | |
277 | except Exception as e: |
|
277 | except Exception as e: | |
278 | self.log.error("Failed to cleanup connection file: %s", e) |
|
278 | self.log.error("Failed to cleanup connection file: %s", e) | |
279 | else: |
|
279 | else: | |
280 | self.log.debug(u"removed %s", f) |
|
280 | self.log.debug(u"removed %s", f) | |
281 |
|
281 | |||
282 | def load_secondary_config(self): |
|
282 | def load_secondary_config(self): | |
283 | """secondary config, loading from JSON and setting defaults""" |
|
283 | """secondary config, loading from JSON and setting defaults""" | |
284 | if self.reuse_files: |
|
284 | if self.reuse_files: | |
285 | try: |
|
285 | try: | |
286 | self.load_config_from_json() |
|
286 | self.load_config_from_json() | |
287 | except (AssertionError,IOError) as e: |
|
287 | except (AssertionError,IOError) as e: | |
288 | self.log.error("Could not load config from JSON: %s" % e) |
|
288 | self.log.error("Could not load config from JSON: %s" % e) | |
289 | else: |
|
289 | else: | |
290 | # successfully loaded config from JSON, and reuse=True |
|
290 | # successfully loaded config from JSON, and reuse=True | |
291 | # no need to wite back the same file |
|
291 | # no need to wite back the same file | |
292 | self.write_connection_files = False |
|
292 | self.write_connection_files = False | |
293 |
|
293 | |||
294 | # switch Session.key default to secure |
|
294 | # switch Session.key default to secure | |
295 | default_secure(self.config) |
|
295 | default_secure(self.config) | |
296 | self.log.debug("Config changed") |
|
296 | self.log.debug("Config changed") | |
297 | self.log.debug(repr(self.config)) |
|
297 | self.log.debug(repr(self.config)) | |
298 |
|
298 | |||
299 | def init_hub(self): |
|
299 | def init_hub(self): | |
300 | c = self.config |
|
300 | c = self.config | |
301 |
|
301 | |||
302 | self.do_import_statements() |
|
302 | self.do_import_statements() | |
303 |
|
303 | |||
304 | try: |
|
304 | try: | |
305 | self.factory = HubFactory(config=c, log=self.log) |
|
305 | self.factory = HubFactory(config=c, log=self.log) | |
306 | # self.start_logging() |
|
306 | # self.start_logging() | |
307 | self.factory.init_hub() |
|
307 | self.factory.init_hub() | |
308 | except TraitError: |
|
308 | except TraitError: | |
309 | raise |
|
309 | raise | |
310 | except Exception: |
|
310 | except Exception: | |
311 | self.log.error("Couldn't construct the Controller", exc_info=True) |
|
311 | self.log.error("Couldn't construct the Controller", exc_info=True) | |
312 | self.exit(1) |
|
312 | self.exit(1) | |
313 |
|
313 | |||
314 | if self.write_connection_files: |
|
314 | if self.write_connection_files: | |
315 | # save to new json config files |
|
315 | # save to new json config files | |
316 | f = self.factory |
|
316 | f = self.factory | |
317 | cdict = {'exec_key' : f.session.key.decode('ascii'), |
|
317 | base = { | |
318 | 'ssh' : self.ssh_server, |
|
318 | 'exec_key' : f.session.key.decode('ascii'), | |
319 | 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport), |
|
319 | 'location' : self.location, | |
320 | 'location' : self.location |
|
320 | 'pack' : f.session.packer, | |
321 | } |
|
321 | 'unpack' : f.session.unpacker, | |
|
322 | } | |||
|
323 | ||||
|
324 | cdict = {'ssh' : self.ssh_server} | |||
|
325 | cdict.update(f.client_info) | |||
|
326 | cdict.update(base) | |||
322 | self.save_connection_dict(self.client_json_file, cdict) |
|
327 | self.save_connection_dict(self.client_json_file, cdict) | |
323 |
|
|
328 | ||
324 | edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport)) |
|
329 | edict = {'ssh' : self.engine_ssh_server} | |
325 |
edict |
|
330 | edict.update(f.engine_info) | |
|
331 | edict.update(base) | |||
326 | self.save_connection_dict(self.engine_json_file, edict) |
|
332 | self.save_connection_dict(self.engine_json_file, edict) | |
327 |
|
333 | |||
328 | def init_schedulers(self): |
|
334 | def init_schedulers(self): | |
329 | children = self.children |
|
335 | children = self.children | |
330 | mq = import_item(str(self.mq_class)) |
|
336 | mq = import_item(str(self.mq_class)) | |
331 |
|
337 | |||
332 | hub = self.factory |
|
338 | hub = self.factory | |
333 | # disambiguate url, in case of * |
|
339 | # disambiguate url, in case of * | |
334 | monitor_url = disambiguate_url(hub.monitor_url) |
|
340 | monitor_url = disambiguate_url(hub.monitor_url) | |
335 | # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url |
|
341 | # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url | |
336 | # IOPub relay (in a Process) |
|
342 | # IOPub relay (in a Process) | |
337 | q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub') |
|
343 | q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub') | |
338 | q.bind_in(hub.client_info['iopub']) |
|
344 | q.bind_in(hub.client_info['iopub']) | |
339 | q.bind_out(hub.engine_info['iopub']) |
|
345 | q.bind_out(hub.engine_info['iopub']) | |
340 | q.setsockopt_out(zmq.SUBSCRIBE, b'') |
|
346 | q.setsockopt_out(zmq.SUBSCRIBE, b'') | |
341 | q.connect_mon(monitor_url) |
|
347 | q.connect_mon(monitor_url) | |
342 | q.daemon=True |
|
348 | q.daemon=True | |
343 | children.append(q) |
|
349 | children.append(q) | |
344 |
|
350 | |||
345 | # Multiplexer Queue (in a Process) |
|
351 | # Multiplexer Queue (in a Process) | |
346 | q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out') |
|
352 | q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out') | |
347 | q.bind_in(hub.client_info['mux']) |
|
353 | q.bind_in(hub.client_info['mux']) | |
348 | q.setsockopt_in(zmq.IDENTITY, b'mux') |
|
354 | q.setsockopt_in(zmq.IDENTITY, b'mux') | |
349 | q.bind_out(hub.engine_info['mux']) |
|
355 | q.bind_out(hub.engine_info['mux']) | |
350 | q.connect_mon(monitor_url) |
|
356 | q.connect_mon(monitor_url) | |
351 | q.daemon=True |
|
357 | q.daemon=True | |
352 | children.append(q) |
|
358 | children.append(q) | |
353 |
|
359 | |||
354 | # Control Queue (in a Process) |
|
360 | # Control Queue (in a Process) | |
355 | q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol') |
|
361 | q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol') | |
356 | q.bind_in(hub.client_info['control']) |
|
362 | q.bind_in(hub.client_info['control']) | |
357 | q.setsockopt_in(zmq.IDENTITY, b'control') |
|
363 | q.setsockopt_in(zmq.IDENTITY, b'control') | |
358 | q.bind_out(hub.engine_info['control']) |
|
364 | q.bind_out(hub.engine_info['control']) | |
359 | q.connect_mon(monitor_url) |
|
365 | q.connect_mon(monitor_url) | |
360 | q.daemon=True |
|
366 | q.daemon=True | |
361 | children.append(q) |
|
367 | children.append(q) | |
362 | try: |
|
368 | try: | |
363 | scheme = self.config.TaskScheduler.scheme_name |
|
369 | scheme = self.config.TaskScheduler.scheme_name | |
364 | except AttributeError: |
|
370 | except AttributeError: | |
365 | scheme = TaskScheduler.scheme_name.get_default_value() |
|
371 | scheme = TaskScheduler.scheme_name.get_default_value() | |
366 | # Task Queue (in a Process) |
|
372 | # Task Queue (in a Process) | |
367 | if scheme == 'pure': |
|
373 | if scheme == 'pure': | |
368 | self.log.warn("task::using pure DEALER Task scheduler") |
|
374 | self.log.warn("task::using pure DEALER Task scheduler") | |
369 | q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask') |
|
375 | q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask') | |
370 | # q.setsockopt_out(zmq.HWM, hub.hwm) |
|
376 | # q.setsockopt_out(zmq.HWM, hub.hwm) | |
371 | q.bind_in(hub.client_info['task'][1]) |
|
377 | q.bind_in(hub.client_info['task'][1]) | |
372 | q.setsockopt_in(zmq.IDENTITY, b'task') |
|
378 | q.setsockopt_in(zmq.IDENTITY, b'task') | |
373 | q.bind_out(hub.engine_info['task']) |
|
379 | q.bind_out(hub.engine_info['task']) | |
374 | q.connect_mon(monitor_url) |
|
380 | q.connect_mon(monitor_url) | |
375 | q.daemon=True |
|
381 | q.daemon=True | |
376 | children.append(q) |
|
382 | children.append(q) | |
377 | elif scheme == 'none': |
|
383 | elif scheme == 'none': | |
378 | self.log.warn("task::using no Task scheduler") |
|
384 | self.log.warn("task::using no Task scheduler") | |
379 |
|
385 | |||
380 | else: |
|
386 | else: | |
381 | self.log.info("task::using Python %s Task scheduler"%scheme) |
|
387 | self.log.info("task::using Python %s Task scheduler"%scheme) | |
382 |
sargs = (hub.client_info['task'] |
|
388 | sargs = (hub.client_info['task'], hub.engine_info['task'], | |
383 | monitor_url, disambiguate_url(hub.client_info['notification'])) |
|
389 | monitor_url, disambiguate_url(hub.client_info['notification'])) | |
384 | kwargs = dict(logname='scheduler', loglevel=self.log_level, |
|
390 | kwargs = dict(logname='scheduler', loglevel=self.log_level, | |
385 | log_url = self.log_url, config=dict(self.config)) |
|
391 | log_url = self.log_url, config=dict(self.config)) | |
386 | if 'Process' in self.mq_class: |
|
392 | if 'Process' in self.mq_class: | |
387 | # run the Python scheduler in a Process |
|
393 | # run the Python scheduler in a Process | |
388 | q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) |
|
394 | q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) | |
389 | q.daemon=True |
|
395 | q.daemon=True | |
390 | children.append(q) |
|
396 | children.append(q) | |
391 | else: |
|
397 | else: | |
392 | # single-threaded Controller |
|
398 | # single-threaded Controller | |
393 | kwargs['in_thread'] = True |
|
399 | kwargs['in_thread'] = True | |
394 | launch_scheduler(*sargs, **kwargs) |
|
400 | launch_scheduler(*sargs, **kwargs) | |
395 |
|
401 | |||
396 | def terminate_children(self): |
|
402 | def terminate_children(self): | |
397 | child_procs = [] |
|
403 | child_procs = [] | |
398 | for child in self.children: |
|
404 | for child in self.children: | |
399 | if isinstance(child, ProcessMonitoredQueue): |
|
405 | if isinstance(child, ProcessMonitoredQueue): | |
400 | child_procs.append(child.launcher) |
|
406 | child_procs.append(child.launcher) | |
401 | elif isinstance(child, Process): |
|
407 | elif isinstance(child, Process): | |
402 | child_procs.append(child) |
|
408 | child_procs.append(child) | |
403 | if child_procs: |
|
409 | if child_procs: | |
404 | self.log.critical("terminating children...") |
|
410 | self.log.critical("terminating children...") | |
405 | for child in child_procs: |
|
411 | for child in child_procs: | |
406 | try: |
|
412 | try: | |
407 | child.terminate() |
|
413 | child.terminate() | |
408 | except OSError: |
|
414 | except OSError: | |
409 | # already dead |
|
415 | # already dead | |
410 | pass |
|
416 | pass | |
411 |
|
417 | |||
412 | def handle_signal(self, sig, frame): |
|
418 | def handle_signal(self, sig, frame): | |
413 | self.log.critical("Received signal %i, shutting down", sig) |
|
419 | self.log.critical("Received signal %i, shutting down", sig) | |
414 | self.terminate_children() |
|
420 | self.terminate_children() | |
415 | self.loop.stop() |
|
421 | self.loop.stop() | |
416 |
|
422 | |||
417 | def init_signal(self): |
|
423 | def init_signal(self): | |
418 | for sig in (SIGINT, SIGABRT, SIGTERM): |
|
424 | for sig in (SIGINT, SIGABRT, SIGTERM): | |
419 | signal(sig, self.handle_signal) |
|
425 | signal(sig, self.handle_signal) | |
420 |
|
426 | |||
421 | def do_import_statements(self): |
|
427 | def do_import_statements(self): | |
422 | statements = self.import_statements |
|
428 | statements = self.import_statements | |
423 | for s in statements: |
|
429 | for s in statements: | |
424 | try: |
|
430 | try: | |
425 | self.log.msg("Executing statement: '%s'" % s) |
|
431 | self.log.msg("Executing statement: '%s'" % s) | |
426 | exec s in globals(), locals() |
|
432 | exec s in globals(), locals() | |
427 | except: |
|
433 | except: | |
428 | self.log.msg("Error running statement: %s" % s) |
|
434 | self.log.msg("Error running statement: %s" % s) | |
429 |
|
435 | |||
430 | def forward_logging(self): |
|
436 | def forward_logging(self): | |
431 | if self.log_url: |
|
437 | if self.log_url: | |
432 | self.log.info("Forwarding logging to %s"%self.log_url) |
|
438 | self.log.info("Forwarding logging to %s"%self.log_url) | |
433 | context = zmq.Context.instance() |
|
439 | context = zmq.Context.instance() | |
434 | lsock = context.socket(zmq.PUB) |
|
440 | lsock = context.socket(zmq.PUB) | |
435 | lsock.connect(self.log_url) |
|
441 | lsock.connect(self.log_url) | |
436 | handler = PUBHandler(lsock) |
|
442 | handler = PUBHandler(lsock) | |
437 | handler.root_topic = 'controller' |
|
443 | handler.root_topic = 'controller' | |
438 | handler.setLevel(self.log_level) |
|
444 | handler.setLevel(self.log_level) | |
439 | self.log.addHandler(handler) |
|
445 | self.log.addHandler(handler) | |
440 |
|
446 | |||
441 | @catch_config_error |
|
447 | @catch_config_error | |
442 | def initialize(self, argv=None): |
|
448 | def initialize(self, argv=None): | |
443 | super(IPControllerApp, self).initialize(argv) |
|
449 | super(IPControllerApp, self).initialize(argv) | |
444 | self.forward_logging() |
|
450 | self.forward_logging() | |
445 | self.load_secondary_config() |
|
451 | self.load_secondary_config() | |
446 | self.init_hub() |
|
452 | self.init_hub() | |
447 | self.init_schedulers() |
|
453 | self.init_schedulers() | |
448 |
|
454 | |||
449 | def start(self): |
|
455 | def start(self): | |
450 | # Start the subprocesses: |
|
456 | # Start the subprocesses: | |
451 | self.factory.start() |
|
457 | self.factory.start() | |
452 | # children must be started before signals are setup, |
|
458 | # children must be started before signals are setup, | |
453 | # otherwise signal-handling will fire multiple times |
|
459 | # otherwise signal-handling will fire multiple times | |
454 | for child in self.children: |
|
460 | for child in self.children: | |
455 | child.start() |
|
461 | child.start() | |
456 | self.init_signal() |
|
462 | self.init_signal() | |
457 |
|
463 | |||
458 | self.write_pid_file(overwrite=True) |
|
464 | self.write_pid_file(overwrite=True) | |
459 |
|
465 | |||
460 | try: |
|
466 | try: | |
461 | self.factory.loop.start() |
|
467 | self.factory.loop.start() | |
462 | except KeyboardInterrupt: |
|
468 | except KeyboardInterrupt: | |
463 | self.log.critical("Interrupted, Exiting...\n") |
|
469 | self.log.critical("Interrupted, Exiting...\n") | |
464 | finally: |
|
470 | finally: | |
465 | self.cleanup_connection_files() |
|
471 | self.cleanup_connection_files() | |
466 |
|
472 | |||
467 |
|
473 | |||
468 |
|
474 | |||
469 | def launch_new_instance(): |
|
475 | def launch_new_instance(): | |
470 | """Create and run the IPython controller""" |
|
476 | """Create and run the IPython controller""" | |
471 | if sys.platform == 'win32': |
|
477 | if sys.platform == 'win32': | |
472 | # make sure we don't get called from a multiprocessing subprocess |
|
478 | # make sure we don't get called from a multiprocessing subprocess | |
473 | # this can result in infinite Controllers being started on Windows |
|
479 | # this can result in infinite Controllers being started on Windows | |
474 | # which doesn't have a proper fork, so multiprocessing is wonky |
|
480 | # which doesn't have a proper fork, so multiprocessing is wonky | |
475 |
|
481 | |||
476 | # this only comes up when IPython has been installed using vanilla |
|
482 | # this only comes up when IPython has been installed using vanilla | |
477 | # setuptools, and *not* distribute. |
|
483 | # setuptools, and *not* distribute. | |
478 | import multiprocessing |
|
484 | import multiprocessing | |
479 | p = multiprocessing.current_process() |
|
485 | p = multiprocessing.current_process() | |
480 | # the main process has name 'MainProcess' |
|
486 | # the main process has name 'MainProcess' | |
481 | # subprocesses will have names like 'Process-1' |
|
487 | # subprocesses will have names like 'Process-1' | |
482 | if p.name != 'MainProcess': |
|
488 | if p.name != 'MainProcess': | |
483 | # we are a subprocess, don't start another Controller! |
|
489 | # we are a subprocess, don't start another Controller! | |
484 | return |
|
490 | return | |
485 | app = IPControllerApp.instance() |
|
491 | app = IPControllerApp.instance() | |
486 | app.initialize() |
|
492 | app.initialize() | |
487 | app.start() |
|
493 | app.start() | |
488 |
|
494 | |||
489 |
|
495 | |||
490 | if __name__ == '__main__': |
|
496 | if __name__ == '__main__': | |
491 | launch_new_instance() |
|
497 | launch_new_instance() |
@@ -1,377 +1,390 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The IPython engine application |
|
4 | The IPython engine application | |
5 |
|
5 | |||
6 | Authors: |
|
6 | Authors: | |
7 |
|
7 | |||
8 | * Brian Granger |
|
8 | * Brian Granger | |
9 | * MinRK |
|
9 | * MinRK | |
10 |
|
10 | |||
11 | """ |
|
11 | """ | |
12 |
|
12 | |||
13 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
14 | # Copyright (C) 2008-2011 The IPython Development Team |
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |
15 | # |
|
15 | # | |
16 | # Distributed under the terms of the BSD License. The full license is in |
|
16 | # Distributed under the terms of the BSD License. The full license is in | |
17 | # the file COPYING, distributed as part of this software. |
|
17 | # the file COPYING, distributed as part of this software. | |
18 | #----------------------------------------------------------------------------- |
|
18 | #----------------------------------------------------------------------------- | |
19 |
|
19 | |||
20 | #----------------------------------------------------------------------------- |
|
20 | #----------------------------------------------------------------------------- | |
21 | # Imports |
|
21 | # Imports | |
22 | #----------------------------------------------------------------------------- |
|
22 | #----------------------------------------------------------------------------- | |
23 |
|
23 | |||
24 | import json |
|
24 | import json | |
25 | import os |
|
25 | import os | |
26 | import sys |
|
26 | import sys | |
27 | import time |
|
27 | import time | |
28 |
|
28 | |||
29 | import zmq |
|
29 | import zmq | |
30 | from zmq.eventloop import ioloop |
|
30 | from zmq.eventloop import ioloop | |
31 |
|
31 | |||
32 | from IPython.core.profiledir import ProfileDir |
|
32 | from IPython.core.profiledir import ProfileDir | |
33 | from IPython.parallel.apps.baseapp import ( |
|
33 | from IPython.parallel.apps.baseapp import ( | |
34 | BaseParallelApplication, |
|
34 | BaseParallelApplication, | |
35 | base_aliases, |
|
35 | base_aliases, | |
36 | base_flags, |
|
36 | base_flags, | |
37 | catch_config_error, |
|
37 | catch_config_error, | |
38 | ) |
|
38 | ) | |
39 | from IPython.zmq.log import EnginePUBHandler |
|
39 | from IPython.zmq.log import EnginePUBHandler | |
40 | from IPython.zmq.ipkernel import Kernel, IPKernelApp |
|
40 | from IPython.zmq.ipkernel import Kernel, IPKernelApp | |
41 | from IPython.zmq.session import ( |
|
41 | from IPython.zmq.session import ( | |
42 | Session, session_aliases, session_flags |
|
42 | Session, session_aliases, session_flags | |
43 | ) |
|
43 | ) | |
44 |
|
44 | |||
45 | from IPython.config.configurable import Configurable |
|
45 | from IPython.config.configurable import Configurable | |
46 |
|
46 | |||
47 | from IPython.parallel.engine.engine import EngineFactory |
|
47 | from IPython.parallel.engine.engine import EngineFactory | |
48 | from IPython.parallel.util import disambiguate_url |
|
48 | from IPython.parallel.util import disambiguate_url | |
49 |
|
49 | |||
50 | from IPython.utils.importstring import import_item |
|
50 | from IPython.utils.importstring import import_item | |
51 | from IPython.utils.py3compat import cast_bytes |
|
51 | from IPython.utils.py3compat import cast_bytes | |
52 | from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float, Instance |
|
52 | from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float, Instance | |
53 |
|
53 | |||
54 |
|
54 | |||
55 | #----------------------------------------------------------------------------- |
|
55 | #----------------------------------------------------------------------------- | |
56 | # Module level variables |
|
56 | # Module level variables | |
57 | #----------------------------------------------------------------------------- |
|
57 | #----------------------------------------------------------------------------- | |
58 |
|
58 | |||
59 | #: The default config file name for this application |
|
59 | #: The default config file name for this application | |
60 | default_config_file_name = u'ipengine_config.py' |
|
60 | default_config_file_name = u'ipengine_config.py' | |
61 |
|
61 | |||
62 | _description = """Start an IPython engine for parallel computing. |
|
62 | _description = """Start an IPython engine for parallel computing. | |
63 |
|
63 | |||
64 | IPython engines run in parallel and perform computations on behalf of a client |
|
64 | IPython engines run in parallel and perform computations on behalf of a client | |
65 | and controller. A controller needs to be started before the engines. The |
|
65 | and controller. A controller needs to be started before the engines. The | |
66 | engine can be configured using command line options or using a cluster |
|
66 | engine can be configured using command line options or using a cluster | |
67 | directory. Cluster directories contain config, log and security files and are |
|
67 | directory. Cluster directories contain config, log and security files and are | |
68 | usually located in your ipython directory and named as "profile_name". |
|
68 | usually located in your ipython directory and named as "profile_name". | |
69 | See the `profile` and `profile-dir` options for details. |
|
69 | See the `profile` and `profile-dir` options for details. | |
70 | """ |
|
70 | """ | |
71 |
|
71 | |||
72 | _examples = """ |
|
72 | _examples = """ | |
73 | ipengine --ip=192.168.0.1 --port=1000 # connect to hub at ip and port |
|
73 | ipengine --ip=192.168.0.1 --port=1000 # connect to hub at ip and port | |
74 | ipengine --log-to-file --log-level=DEBUG # log to a file with DEBUG verbosity |
|
74 | ipengine --log-to-file --log-level=DEBUG # log to a file with DEBUG verbosity | |
75 | """ |
|
75 | """ | |
76 |
|
76 | |||
77 | #----------------------------------------------------------------------------- |
|
77 | #----------------------------------------------------------------------------- | |
78 | # MPI configuration |
|
78 | # MPI configuration | |
79 | #----------------------------------------------------------------------------- |
|
79 | #----------------------------------------------------------------------------- | |
80 |
|
80 | |||
81 | mpi4py_init = """from mpi4py import MPI as mpi |
|
81 | mpi4py_init = """from mpi4py import MPI as mpi | |
82 | mpi.size = mpi.COMM_WORLD.Get_size() |
|
82 | mpi.size = mpi.COMM_WORLD.Get_size() | |
83 | mpi.rank = mpi.COMM_WORLD.Get_rank() |
|
83 | mpi.rank = mpi.COMM_WORLD.Get_rank() | |
84 | """ |
|
84 | """ | |
85 |
|
85 | |||
86 |
|
86 | |||
87 | pytrilinos_init = """from PyTrilinos import Epetra |
|
87 | pytrilinos_init = """from PyTrilinos import Epetra | |
88 | class SimpleStruct: |
|
88 | class SimpleStruct: | |
89 | pass |
|
89 | pass | |
90 | mpi = SimpleStruct() |
|
90 | mpi = SimpleStruct() | |
91 | mpi.rank = 0 |
|
91 | mpi.rank = 0 | |
92 | mpi.size = 0 |
|
92 | mpi.size = 0 | |
93 | """ |
|
93 | """ | |
94 |
|
94 | |||
95 | class MPI(Configurable): |
|
95 | class MPI(Configurable): | |
96 | """Configurable for MPI initialization""" |
|
96 | """Configurable for MPI initialization""" | |
97 | use = Unicode('', config=True, |
|
97 | use = Unicode('', config=True, | |
98 | help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).' |
|
98 | help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).' | |
99 | ) |
|
99 | ) | |
100 |
|
100 | |||
101 | def _use_changed(self, name, old, new): |
|
101 | def _use_changed(self, name, old, new): | |
102 | # load default init script if it's not set |
|
102 | # load default init script if it's not set | |
103 | if not self.init_script: |
|
103 | if not self.init_script: | |
104 | self.init_script = self.default_inits.get(new, '') |
|
104 | self.init_script = self.default_inits.get(new, '') | |
105 |
|
105 | |||
106 | init_script = Unicode('', config=True, |
|
106 | init_script = Unicode('', config=True, | |
107 | help="Initialization code for MPI") |
|
107 | help="Initialization code for MPI") | |
108 |
|
108 | |||
109 | default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init}, |
|
109 | default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init}, | |
110 | config=True) |
|
110 | config=True) | |
111 |
|
111 | |||
112 |
|
112 | |||
113 | #----------------------------------------------------------------------------- |
|
113 | #----------------------------------------------------------------------------- | |
114 | # Main application |
|
114 | # Main application | |
115 | #----------------------------------------------------------------------------- |
|
115 | #----------------------------------------------------------------------------- | |
116 | aliases = dict( |
|
116 | aliases = dict( | |
117 | file = 'IPEngineApp.url_file', |
|
117 | file = 'IPEngineApp.url_file', | |
118 | c = 'IPEngineApp.startup_command', |
|
118 | c = 'IPEngineApp.startup_command', | |
119 | s = 'IPEngineApp.startup_script', |
|
119 | s = 'IPEngineApp.startup_script', | |
120 |
|
120 | |||
121 | url = 'EngineFactory.url', |
|
121 | url = 'EngineFactory.url', | |
122 | ssh = 'EngineFactory.sshserver', |
|
122 | ssh = 'EngineFactory.sshserver', | |
123 | sshkey = 'EngineFactory.sshkey', |
|
123 | sshkey = 'EngineFactory.sshkey', | |
124 | ip = 'EngineFactory.ip', |
|
124 | ip = 'EngineFactory.ip', | |
125 | transport = 'EngineFactory.transport', |
|
125 | transport = 'EngineFactory.transport', | |
126 | port = 'EngineFactory.regport', |
|
126 | port = 'EngineFactory.regport', | |
127 | location = 'EngineFactory.location', |
|
127 | location = 'EngineFactory.location', | |
128 |
|
128 | |||
129 | timeout = 'EngineFactory.timeout', |
|
129 | timeout = 'EngineFactory.timeout', | |
130 |
|
130 | |||
131 | mpi = 'MPI.use', |
|
131 | mpi = 'MPI.use', | |
132 |
|
132 | |||
133 | ) |
|
133 | ) | |
134 | aliases.update(base_aliases) |
|
134 | aliases.update(base_aliases) | |
135 | aliases.update(session_aliases) |
|
135 | aliases.update(session_aliases) | |
136 | flags = {} |
|
136 | flags = {} | |
137 | flags.update(base_flags) |
|
137 | flags.update(base_flags) | |
138 | flags.update(session_flags) |
|
138 | flags.update(session_flags) | |
139 |
|
139 | |||
140 | class IPEngineApp(BaseParallelApplication): |
|
140 | class IPEngineApp(BaseParallelApplication): | |
141 |
|
141 | |||
142 | name = 'ipengine' |
|
142 | name = 'ipengine' | |
143 | description = _description |
|
143 | description = _description | |
144 | examples = _examples |
|
144 | examples = _examples | |
145 | config_file_name = Unicode(default_config_file_name) |
|
145 | config_file_name = Unicode(default_config_file_name) | |
146 | classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI]) |
|
146 | classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI]) | |
147 |
|
147 | |||
148 | startup_script = Unicode(u'', config=True, |
|
148 | startup_script = Unicode(u'', config=True, | |
149 | help='specify a script to be run at startup') |
|
149 | help='specify a script to be run at startup') | |
150 | startup_command = Unicode('', config=True, |
|
150 | startup_command = Unicode('', config=True, | |
151 | help='specify a command to be run at startup') |
|
151 | help='specify a command to be run at startup') | |
152 |
|
152 | |||
153 | url_file = Unicode(u'', config=True, |
|
153 | url_file = Unicode(u'', config=True, | |
154 | help="""The full location of the file containing the connection information for |
|
154 | help="""The full location of the file containing the connection information for | |
155 | the controller. If this is not given, the file must be in the |
|
155 | the controller. If this is not given, the file must be in the | |
156 | security directory of the cluster directory. This location is |
|
156 | security directory of the cluster directory. This location is | |
157 | resolved using the `profile` or `profile_dir` options.""", |
|
157 | resolved using the `profile` or `profile_dir` options.""", | |
158 | ) |
|
158 | ) | |
159 | wait_for_url_file = Float(5, config=True, |
|
159 | wait_for_url_file = Float(5, config=True, | |
160 | help="""The maximum number of seconds to wait for url_file to exist. |
|
160 | help="""The maximum number of seconds to wait for url_file to exist. | |
161 | This is useful for batch-systems and shared-filesystems where the |
|
161 | This is useful for batch-systems and shared-filesystems where the | |
162 | controller and engine are started at the same time and it |
|
162 | controller and engine are started at the same time and it | |
163 | may take a moment for the controller to write the connector files.""") |
|
163 | may take a moment for the controller to write the connector files.""") | |
164 |
|
164 | |||
165 | url_file_name = Unicode(u'ipcontroller-engine.json', config=True) |
|
165 | url_file_name = Unicode(u'ipcontroller-engine.json', config=True) | |
166 |
|
166 | |||
167 | def _cluster_id_changed(self, name, old, new): |
|
167 | def _cluster_id_changed(self, name, old, new): | |
168 | if new: |
|
168 | if new: | |
169 | base = 'ipcontroller-%s' % new |
|
169 | base = 'ipcontroller-%s' % new | |
170 | else: |
|
170 | else: | |
171 | base = 'ipcontroller' |
|
171 | base = 'ipcontroller' | |
172 | self.url_file_name = "%s-engine.json" % base |
|
172 | self.url_file_name = "%s-engine.json" % base | |
173 |
|
173 | |||
174 | log_url = Unicode('', config=True, |
|
174 | log_url = Unicode('', config=True, | |
175 | help="""The URL for the iploggerapp instance, for forwarding |
|
175 | help="""The URL for the iploggerapp instance, for forwarding | |
176 | logging to a central location.""") |
|
176 | logging to a central location.""") | |
177 |
|
177 | |||
178 | # an IPKernelApp instance, used to setup listening for shell frontends |
|
178 | # an IPKernelApp instance, used to setup listening for shell frontends | |
179 | kernel_app = Instance(IPKernelApp) |
|
179 | kernel_app = Instance(IPKernelApp) | |
180 |
|
180 | |||
181 | aliases = Dict(aliases) |
|
181 | aliases = Dict(aliases) | |
182 | flags = Dict(flags) |
|
182 | flags = Dict(flags) | |
183 |
|
183 | |||
184 | @property |
|
184 | @property | |
185 | def kernel(self): |
|
185 | def kernel(self): | |
186 | """allow access to the Kernel object, so I look like IPKernelApp""" |
|
186 | """allow access to the Kernel object, so I look like IPKernelApp""" | |
187 | return self.engine.kernel |
|
187 | return self.engine.kernel | |
188 |
|
188 | |||
189 | def find_url_file(self): |
|
189 | def find_url_file(self): | |
190 | """Set the url file. |
|
190 | """Set the url file. | |
191 |
|
191 | |||
192 | Here we don't try to actually see if it exists for is valid as that |
|
192 | Here we don't try to actually see if it exists for is valid as that | |
193 | is hadled by the connection logic. |
|
193 | is hadled by the connection logic. | |
194 | """ |
|
194 | """ | |
195 | config = self.config |
|
195 | config = self.config | |
196 | # Find the actual controller key file |
|
196 | # Find the actual controller key file | |
197 | if not self.url_file: |
|
197 | if not self.url_file: | |
198 | self.url_file = os.path.join( |
|
198 | self.url_file = os.path.join( | |
199 | self.profile_dir.security_dir, |
|
199 | self.profile_dir.security_dir, | |
200 | self.url_file_name |
|
200 | self.url_file_name | |
201 | ) |
|
201 | ) | |
202 |
|
202 | |||
203 | def load_connector_file(self): |
|
203 | def load_connector_file(self): | |
204 | """load config from a JSON connector file, |
|
204 | """load config from a JSON connector file, | |
205 | at a *lower* priority than command-line/config files. |
|
205 | at a *lower* priority than command-line/config files. | |
206 | """ |
|
206 | """ | |
207 |
|
207 | |||
208 | self.log.info("Loading url_file %r", self.url_file) |
|
208 | self.log.info("Loading url_file %r", self.url_file) | |
209 | config = self.config |
|
209 | config = self.config | |
210 |
|
210 | |||
211 | with open(self.url_file) as f: |
|
211 | with open(self.url_file) as f: | |
212 | d = json.loads(f.read()) |
|
212 | d = json.loads(f.read()) | |
213 |
|
213 | |||
214 | if 'exec_key' in d: |
|
214 | # allow hand-override of location for disambiguation | |
215 | config.Session.key = cast_bytes(d['exec_key']) |
|
215 | # and ssh-server | |
216 |
|
||||
217 | try: |
|
216 | try: | |
218 | config.EngineFactory.location |
|
217 | config.EngineFactory.location | |
219 | except AttributeError: |
|
218 | except AttributeError: | |
220 | config.EngineFactory.location = d['location'] |
|
219 | config.EngineFactory.location = d['location'] | |
221 |
|
220 | |||
222 | d['url'] = disambiguate_url(d['url'], config.EngineFactory.location) |
|
|||
223 | try: |
|
|||
224 | config.EngineFactory.url |
|
|||
225 | except AttributeError: |
|
|||
226 | config.EngineFactory.url = d['url'] |
|
|||
227 |
|
||||
228 | try: |
|
221 | try: | |
229 | config.EngineFactory.sshserver |
|
222 | config.EngineFactory.sshserver | |
230 | except AttributeError: |
|
223 | except AttributeError: | |
231 |
config.EngineFactory.sshserver = d |
|
224 | config.EngineFactory.sshserver = d.get('ssh') | |
|
225 | ||||
|
226 | location = config.EngineFactory.location | |||
|
227 | ||||
|
228 | for key in ('registration', 'hb_ping', 'hb_pong', 'mux', 'task', 'control'): | |||
|
229 | d[key] = disambiguate_url(d[key], location) | |||
|
230 | ||||
|
231 | # DO NOT allow override of basic URLs, serialization, or exec_key | |||
|
232 | # JSON file takes top priority there | |||
|
233 | config.Session.key = asbytes(d['exec_key']) | |||
|
234 | ||||
|
235 | config.EngineFactory.url = d['registration'] | |||
|
236 | ||||
|
237 | config.Session.packer = d['pack'] | |||
|
238 | config.Session.unpacker = d['unpack'] | |||
|
239 | ||||
|
240 | self.log.debug("Config changed:") | |||
|
241 | self.log.debug("%r", config) | |||
|
242 | self.connection_info = d | |||
232 |
|
243 | |||
233 | def bind_kernel(self, **kwargs): |
|
244 | def bind_kernel(self, **kwargs): | |
234 | """Promote engine to listening kernel, accessible to frontends.""" |
|
245 | """Promote engine to listening kernel, accessible to frontends.""" | |
235 | if self.kernel_app is not None: |
|
246 | if self.kernel_app is not None: | |
236 | return |
|
247 | return | |
237 |
|
248 | |||
238 | self.log.info("Opening ports for direct connections as an IPython kernel") |
|
249 | self.log.info("Opening ports for direct connections as an IPython kernel") | |
239 |
|
250 | |||
240 | kernel = self.kernel |
|
251 | kernel = self.kernel | |
241 |
|
252 | |||
242 | kwargs.setdefault('config', self.config) |
|
253 | kwargs.setdefault('config', self.config) | |
243 | kwargs.setdefault('log', self.log) |
|
254 | kwargs.setdefault('log', self.log) | |
244 | kwargs.setdefault('profile_dir', self.profile_dir) |
|
255 | kwargs.setdefault('profile_dir', self.profile_dir) | |
245 | kwargs.setdefault('session', self.engine.session) |
|
256 | kwargs.setdefault('session', self.engine.session) | |
246 |
|
257 | |||
247 | app = self.kernel_app = IPKernelApp(**kwargs) |
|
258 | app = self.kernel_app = IPKernelApp(**kwargs) | |
248 |
|
259 | |||
249 | # allow IPKernelApp.instance(): |
|
260 | # allow IPKernelApp.instance(): | |
250 | IPKernelApp._instance = app |
|
261 | IPKernelApp._instance = app | |
251 |
|
262 | |||
252 | app.init_connection_file() |
|
263 | app.init_connection_file() | |
253 | # relevant contents of init_sockets: |
|
264 | # relevant contents of init_sockets: | |
254 |
|
265 | |||
255 | app.shell_port = app._bind_socket(kernel.shell_streams[0], app.shell_port) |
|
266 | app.shell_port = app._bind_socket(kernel.shell_streams[0], app.shell_port) | |
256 | app.log.debug("shell ROUTER Channel on port: %i", app.shell_port) |
|
267 | app.log.debug("shell ROUTER Channel on port: %i", app.shell_port) | |
257 |
|
268 | |||
258 | app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port) |
|
269 | app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port) | |
259 | app.log.debug("iopub PUB Channel on port: %i", app.iopub_port) |
|
270 | app.log.debug("iopub PUB Channel on port: %i", app.iopub_port) | |
260 |
|
271 | |||
261 | kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER) |
|
272 | kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER) | |
262 | app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port) |
|
273 | app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port) | |
263 | app.log.debug("stdin ROUTER Channel on port: %i", app.stdin_port) |
|
274 | app.log.debug("stdin ROUTER Channel on port: %i", app.stdin_port) | |
264 |
|
275 | |||
265 | # start the heartbeat, and log connection info: |
|
276 | # start the heartbeat, and log connection info: | |
266 |
|
277 | |||
267 | app.init_heartbeat() |
|
278 | app.init_heartbeat() | |
268 |
|
279 | |||
269 | app.log_connection_info() |
|
280 | app.log_connection_info() | |
270 | app.write_connection_file() |
|
281 | app.write_connection_file() | |
271 |
|
282 | |||
272 |
|
283 | |||
273 | def init_engine(self): |
|
284 | def init_engine(self): | |
274 | # This is the working dir by now. |
|
285 | # This is the working dir by now. | |
275 | sys.path.insert(0, '') |
|
286 | sys.path.insert(0, '') | |
276 | config = self.config |
|
287 | config = self.config | |
277 | # print config |
|
288 | # print config | |
278 | self.find_url_file() |
|
289 | self.find_url_file() | |
279 |
|
290 | |||
280 | # was the url manually specified? |
|
291 | # was the url manually specified? | |
281 | keys = set(self.config.EngineFactory.keys()) |
|
292 | keys = set(self.config.EngineFactory.keys()) | |
282 | keys = keys.union(set(self.config.RegistrationFactory.keys())) |
|
293 | keys = keys.union(set(self.config.RegistrationFactory.keys())) | |
283 |
|
294 | |||
284 | if keys.intersection(set(['ip', 'url', 'port'])): |
|
295 | if keys.intersection(set(['ip', 'url', 'port'])): | |
285 | # Connection info was specified, don't wait for the file |
|
296 | # Connection info was specified, don't wait for the file | |
286 | url_specified = True |
|
297 | url_specified = True | |
287 | self.wait_for_url_file = 0 |
|
298 | self.wait_for_url_file = 0 | |
288 | else: |
|
299 | else: | |
289 | url_specified = False |
|
300 | url_specified = False | |
290 |
|
301 | |||
291 | if self.wait_for_url_file and not os.path.exists(self.url_file): |
|
302 | if self.wait_for_url_file and not os.path.exists(self.url_file): | |
292 | self.log.warn("url_file %r not found", self.url_file) |
|
303 | self.log.warn("url_file %r not found", self.url_file) | |
293 | self.log.warn("Waiting up to %.1f seconds for it to arrive.", self.wait_for_url_file) |
|
304 | self.log.warn("Waiting up to %.1f seconds for it to arrive.", self.wait_for_url_file) | |
294 | tic = time.time() |
|
305 | tic = time.time() | |
295 | while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file): |
|
306 | while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file): | |
296 | # wait for url_file to exist, or until time limit |
|
307 | # wait for url_file to exist, or until time limit | |
297 | time.sleep(0.1) |
|
308 | time.sleep(0.1) | |
298 |
|
309 | |||
299 | if os.path.exists(self.url_file): |
|
310 | if os.path.exists(self.url_file): | |
300 | self.load_connector_file() |
|
311 | self.load_connector_file() | |
301 | elif not url_specified: |
|
312 | elif not url_specified: | |
302 | self.log.fatal("Fatal: url file never arrived: %s", self.url_file) |
|
313 | self.log.fatal("Fatal: url file never arrived: %s", self.url_file) | |
303 | self.exit(1) |
|
314 | self.exit(1) | |
304 |
|
315 | |||
305 |
|
316 | |||
306 | try: |
|
317 | try: | |
307 | exec_lines = config.Kernel.exec_lines |
|
318 | exec_lines = config.Kernel.exec_lines | |
308 | except AttributeError: |
|
319 | except AttributeError: | |
309 | config.Kernel.exec_lines = [] |
|
320 | config.Kernel.exec_lines = [] | |
310 | exec_lines = config.Kernel.exec_lines |
|
321 | exec_lines = config.Kernel.exec_lines | |
311 |
|
322 | |||
312 | if self.startup_script: |
|
323 | if self.startup_script: | |
313 | enc = sys.getfilesystemencoding() or 'utf8' |
|
324 | enc = sys.getfilesystemencoding() or 'utf8' | |
314 | cmd="execfile(%r)" % self.startup_script.encode(enc) |
|
325 | cmd="execfile(%r)" % self.startup_script.encode(enc) | |
315 | exec_lines.append(cmd) |
|
326 | exec_lines.append(cmd) | |
316 | if self.startup_command: |
|
327 | if self.startup_command: | |
317 | exec_lines.append(self.startup_command) |
|
328 | exec_lines.append(self.startup_command) | |
318 |
|
329 | |||
319 | # Create the underlying shell class and Engine |
|
330 | # Create the underlying shell class and Engine | |
320 | # shell_class = import_item(self.master_config.Global.shell_class) |
|
331 | # shell_class = import_item(self.master_config.Global.shell_class) | |
321 | # print self.config |
|
332 | # print self.config | |
322 | try: |
|
333 | try: | |
323 |
self.engine = EngineFactory(config=config, log=self.log |
|
334 | self.engine = EngineFactory(config=config, log=self.log, | |
|
335 | connection_info=self.connection_info, | |||
|
336 | ) | |||
324 | except: |
|
337 | except: | |
325 | self.log.error("Couldn't start the Engine", exc_info=True) |
|
338 | self.log.error("Couldn't start the Engine", exc_info=True) | |
326 | self.exit(1) |
|
339 | self.exit(1) | |
327 |
|
340 | |||
328 | def forward_logging(self): |
|
341 | def forward_logging(self): | |
329 | if self.log_url: |
|
342 | if self.log_url: | |
330 | self.log.info("Forwarding logging to %s", self.log_url) |
|
343 | self.log.info("Forwarding logging to %s", self.log_url) | |
331 | context = self.engine.context |
|
344 | context = self.engine.context | |
332 | lsock = context.socket(zmq.PUB) |
|
345 | lsock = context.socket(zmq.PUB) | |
333 | lsock.connect(self.log_url) |
|
346 | lsock.connect(self.log_url) | |
334 | handler = EnginePUBHandler(self.engine, lsock) |
|
347 | handler = EnginePUBHandler(self.engine, lsock) | |
335 | handler.setLevel(self.log_level) |
|
348 | handler.setLevel(self.log_level) | |
336 | self.log.addHandler(handler) |
|
349 | self.log.addHandler(handler) | |
337 |
|
350 | |||
338 | def init_mpi(self): |
|
351 | def init_mpi(self): | |
339 | global mpi |
|
352 | global mpi | |
340 | self.mpi = MPI(config=self.config) |
|
353 | self.mpi = MPI(config=self.config) | |
341 |
|
354 | |||
342 | mpi_import_statement = self.mpi.init_script |
|
355 | mpi_import_statement = self.mpi.init_script | |
343 | if mpi_import_statement: |
|
356 | if mpi_import_statement: | |
344 | try: |
|
357 | try: | |
345 | self.log.info("Initializing MPI:") |
|
358 | self.log.info("Initializing MPI:") | |
346 | self.log.info(mpi_import_statement) |
|
359 | self.log.info(mpi_import_statement) | |
347 | exec mpi_import_statement in globals() |
|
360 | exec mpi_import_statement in globals() | |
348 | except: |
|
361 | except: | |
349 | mpi = None |
|
362 | mpi = None | |
350 | else: |
|
363 | else: | |
351 | mpi = None |
|
364 | mpi = None | |
352 |
|
365 | |||
353 | @catch_config_error |
|
366 | @catch_config_error | |
354 | def initialize(self, argv=None): |
|
367 | def initialize(self, argv=None): | |
355 | super(IPEngineApp, self).initialize(argv) |
|
368 | super(IPEngineApp, self).initialize(argv) | |
356 | self.init_mpi() |
|
369 | self.init_mpi() | |
357 | self.init_engine() |
|
370 | self.init_engine() | |
358 | self.forward_logging() |
|
371 | self.forward_logging() | |
359 |
|
372 | |||
360 | def start(self): |
|
373 | def start(self): | |
361 | self.engine.start() |
|
374 | self.engine.start() | |
362 | try: |
|
375 | try: | |
363 | self.engine.loop.start() |
|
376 | self.engine.loop.start() | |
364 | except KeyboardInterrupt: |
|
377 | except KeyboardInterrupt: | |
365 | self.log.critical("Engine Interrupted, shutting down...\n") |
|
378 | self.log.critical("Engine Interrupted, shutting down...\n") | |
366 |
|
379 | |||
367 |
|
380 | |||
368 | def launch_new_instance(): |
|
381 | def launch_new_instance(): | |
369 | """Create and run the IPython engine""" |
|
382 | """Create and run the IPython engine""" | |
370 | app = IPEngineApp.instance() |
|
383 | app = IPEngineApp.instance() | |
371 | app.initialize() |
|
384 | app.initialize() | |
372 | app.start() |
|
385 | app.start() | |
373 |
|
386 | |||
374 |
|
387 | |||
375 | if __name__ == '__main__': |
|
388 | if __name__ == '__main__': | |
376 | launch_new_instance() |
|
389 | launch_new_instance() | |
377 |
|
390 |
@@ -1,1713 +1,1692 b'' | |||||
1 | """A semi-synchronous Client for the ZMQ cluster |
|
1 | """A semi-synchronous Client for the ZMQ cluster | |
2 |
|
2 | |||
3 | Authors: |
|
3 | Authors: | |
4 |
|
4 | |||
5 | * MinRK |
|
5 | * MinRK | |
6 | """ |
|
6 | """ | |
7 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
8 | # Copyright (C) 2010-2011 The IPython Development Team |
|
8 | # Copyright (C) 2010-2011 The IPython Development Team | |
9 | # |
|
9 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. | |
12 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
13 |
|
13 | |||
14 | #----------------------------------------------------------------------------- |
|
14 | #----------------------------------------------------------------------------- | |
15 | # Imports |
|
15 | # Imports | |
16 | #----------------------------------------------------------------------------- |
|
16 | #----------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | import os |
|
18 | import os | |
19 | import json |
|
19 | import json | |
20 | import sys |
|
20 | import sys | |
21 | from threading import Thread, Event |
|
21 | from threading import Thread, Event | |
22 | import time |
|
22 | import time | |
23 | import warnings |
|
23 | import warnings | |
24 | from datetime import datetime |
|
24 | from datetime import datetime | |
25 | from getpass import getpass |
|
25 | from getpass import getpass | |
26 | from pprint import pprint |
|
26 | from pprint import pprint | |
27 |
|
27 | |||
28 | pjoin = os.path.join |
|
28 | pjoin = os.path.join | |
29 |
|
29 | |||
30 | import zmq |
|
30 | import zmq | |
31 | # from zmq.eventloop import ioloop, zmqstream |
|
31 | # from zmq.eventloop import ioloop, zmqstream | |
32 |
|
32 | |||
33 | from IPython.config.configurable import MultipleInstanceError |
|
33 | from IPython.config.configurable import MultipleInstanceError | |
34 | from IPython.core.application import BaseIPythonApplication |
|
34 | from IPython.core.application import BaseIPythonApplication | |
35 | from IPython.core.profiledir import ProfileDir, ProfileDirError |
|
35 | from IPython.core.profiledir import ProfileDir, ProfileDirError | |
36 |
|
36 | |||
37 | from IPython.utils.coloransi import TermColors |
|
37 | from IPython.utils.coloransi import TermColors | |
38 | from IPython.utils.jsonutil import rekey |
|
38 | from IPython.utils.jsonutil import rekey | |
39 | from IPython.utils.localinterfaces import LOCAL_IPS |
|
39 | from IPython.utils.localinterfaces import LOCAL_IPS | |
40 | from IPython.utils.path import get_ipython_dir |
|
40 | from IPython.utils.path import get_ipython_dir | |
41 | from IPython.utils.py3compat import cast_bytes |
|
41 | from IPython.utils.py3compat import cast_bytes | |
42 | from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode, |
|
42 | from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode, | |
43 | Dict, List, Bool, Set, Any) |
|
43 | Dict, List, Bool, Set, Any) | |
44 | from IPython.external.decorator import decorator |
|
44 | from IPython.external.decorator import decorator | |
45 | from IPython.external.ssh import tunnel |
|
45 | from IPython.external.ssh import tunnel | |
46 |
|
46 | |||
47 | from IPython.parallel import Reference |
|
47 | from IPython.parallel import Reference | |
48 | from IPython.parallel import error |
|
48 | from IPython.parallel import error | |
49 | from IPython.parallel import util |
|
49 | from IPython.parallel import util | |
50 |
|
50 | |||
51 | from IPython.zmq.session import Session, Message |
|
51 | from IPython.zmq.session import Session, Message | |
52 |
|
52 | |||
53 | from .asyncresult import AsyncResult, AsyncHubResult |
|
53 | from .asyncresult import AsyncResult, AsyncHubResult | |
54 | from .view import DirectView, LoadBalancedView |
|
54 | from .view import DirectView, LoadBalancedView | |
55 |
|
55 | |||
56 | if sys.version_info[0] >= 3: |
|
56 | if sys.version_info[0] >= 3: | |
57 | # xrange is used in a couple 'isinstance' tests in py2 |
|
57 | # xrange is used in a couple 'isinstance' tests in py2 | |
58 | # should be just 'range' in 3k |
|
58 | # should be just 'range' in 3k | |
59 | xrange = range |
|
59 | xrange = range | |
60 |
|
60 | |||
61 | #-------------------------------------------------------------------------- |
|
61 | #-------------------------------------------------------------------------- | |
62 | # Decorators for Client methods |
|
62 | # Decorators for Client methods | |
63 | #-------------------------------------------------------------------------- |
|
63 | #-------------------------------------------------------------------------- | |
64 |
|
64 | |||
65 | @decorator |
|
65 | @decorator | |
66 | def spin_first(f, self, *args, **kwargs): |
|
66 | def spin_first(f, self, *args, **kwargs): | |
67 | """Call spin() to sync state prior to calling the method.""" |
|
67 | """Call spin() to sync state prior to calling the method.""" | |
68 | self.spin() |
|
68 | self.spin() | |
69 | return f(self, *args, **kwargs) |
|
69 | return f(self, *args, **kwargs) | |
70 |
|
70 | |||
71 |
|
71 | |||
72 | #-------------------------------------------------------------------------- |
|
72 | #-------------------------------------------------------------------------- | |
73 | # Classes |
|
73 | # Classes | |
74 | #-------------------------------------------------------------------------- |
|
74 | #-------------------------------------------------------------------------- | |
75 |
|
75 | |||
76 |
|
76 | |||
77 | class ExecuteReply(object): |
|
77 | class ExecuteReply(object): | |
78 | """wrapper for finished Execute results""" |
|
78 | """wrapper for finished Execute results""" | |
79 | def __init__(self, msg_id, content, metadata): |
|
79 | def __init__(self, msg_id, content, metadata): | |
80 | self.msg_id = msg_id |
|
80 | self.msg_id = msg_id | |
81 | self._content = content |
|
81 | self._content = content | |
82 | self.execution_count = content['execution_count'] |
|
82 | self.execution_count = content['execution_count'] | |
83 | self.metadata = metadata |
|
83 | self.metadata = metadata | |
84 |
|
84 | |||
85 | def __getitem__(self, key): |
|
85 | def __getitem__(self, key): | |
86 | return self.metadata[key] |
|
86 | return self.metadata[key] | |
87 |
|
87 | |||
88 | def __getattr__(self, key): |
|
88 | def __getattr__(self, key): | |
89 | if key not in self.metadata: |
|
89 | if key not in self.metadata: | |
90 | raise AttributeError(key) |
|
90 | raise AttributeError(key) | |
91 | return self.metadata[key] |
|
91 | return self.metadata[key] | |
92 |
|
92 | |||
93 | def __repr__(self): |
|
93 | def __repr__(self): | |
94 | pyout = self.metadata['pyout'] or {'data':{}} |
|
94 | pyout = self.metadata['pyout'] or {'data':{}} | |
95 | text_out = pyout['data'].get('text/plain', '') |
|
95 | text_out = pyout['data'].get('text/plain', '') | |
96 | if len(text_out) > 32: |
|
96 | if len(text_out) > 32: | |
97 | text_out = text_out[:29] + '...' |
|
97 | text_out = text_out[:29] + '...' | |
98 |
|
98 | |||
99 | return "<ExecuteReply[%i]: %s>" % (self.execution_count, text_out) |
|
99 | return "<ExecuteReply[%i]: %s>" % (self.execution_count, text_out) | |
100 |
|
100 | |||
101 | def _repr_pretty_(self, p, cycle): |
|
101 | def _repr_pretty_(self, p, cycle): | |
102 | pyout = self.metadata['pyout'] or {'data':{}} |
|
102 | pyout = self.metadata['pyout'] or {'data':{}} | |
103 | text_out = pyout['data'].get('text/plain', '') |
|
103 | text_out = pyout['data'].get('text/plain', '') | |
104 |
|
104 | |||
105 | if not text_out: |
|
105 | if not text_out: | |
106 | return |
|
106 | return | |
107 |
|
107 | |||
108 | try: |
|
108 | try: | |
109 | ip = get_ipython() |
|
109 | ip = get_ipython() | |
110 | except NameError: |
|
110 | except NameError: | |
111 | colors = "NoColor" |
|
111 | colors = "NoColor" | |
112 | else: |
|
112 | else: | |
113 | colors = ip.colors |
|
113 | colors = ip.colors | |
114 |
|
114 | |||
115 | if colors == "NoColor": |
|
115 | if colors == "NoColor": | |
116 | out = normal = "" |
|
116 | out = normal = "" | |
117 | else: |
|
117 | else: | |
118 | out = TermColors.Red |
|
118 | out = TermColors.Red | |
119 | normal = TermColors.Normal |
|
119 | normal = TermColors.Normal | |
120 |
|
120 | |||
121 | if '\n' in text_out and not text_out.startswith('\n'): |
|
121 | if '\n' in text_out and not text_out.startswith('\n'): | |
122 | # add newline for multiline reprs |
|
122 | # add newline for multiline reprs | |
123 | text_out = '\n' + text_out |
|
123 | text_out = '\n' + text_out | |
124 |
|
124 | |||
125 | p.text( |
|
125 | p.text( | |
126 | out + u'Out[%i:%i]: ' % ( |
|
126 | out + u'Out[%i:%i]: ' % ( | |
127 | self.metadata['engine_id'], self.execution_count |
|
127 | self.metadata['engine_id'], self.execution_count | |
128 | ) + normal + text_out |
|
128 | ) + normal + text_out | |
129 | ) |
|
129 | ) | |
130 |
|
130 | |||
131 | def _repr_html_(self): |
|
131 | def _repr_html_(self): | |
132 | pyout = self.metadata['pyout'] or {'data':{}} |
|
132 | pyout = self.metadata['pyout'] or {'data':{}} | |
133 | return pyout['data'].get("text/html") |
|
133 | return pyout['data'].get("text/html") | |
134 |
|
134 | |||
135 | def _repr_latex_(self): |
|
135 | def _repr_latex_(self): | |
136 | pyout = self.metadata['pyout'] or {'data':{}} |
|
136 | pyout = self.metadata['pyout'] or {'data':{}} | |
137 | return pyout['data'].get("text/latex") |
|
137 | return pyout['data'].get("text/latex") | |
138 |
|
138 | |||
139 | def _repr_json_(self): |
|
139 | def _repr_json_(self): | |
140 | pyout = self.metadata['pyout'] or {'data':{}} |
|
140 | pyout = self.metadata['pyout'] or {'data':{}} | |
141 | return pyout['data'].get("application/json") |
|
141 | return pyout['data'].get("application/json") | |
142 |
|
142 | |||
143 | def _repr_javascript_(self): |
|
143 | def _repr_javascript_(self): | |
144 | pyout = self.metadata['pyout'] or {'data':{}} |
|
144 | pyout = self.metadata['pyout'] or {'data':{}} | |
145 | return pyout['data'].get("application/javascript") |
|
145 | return pyout['data'].get("application/javascript") | |
146 |
|
146 | |||
147 | def _repr_png_(self): |
|
147 | def _repr_png_(self): | |
148 | pyout = self.metadata['pyout'] or {'data':{}} |
|
148 | pyout = self.metadata['pyout'] or {'data':{}} | |
149 | return pyout['data'].get("image/png") |
|
149 | return pyout['data'].get("image/png") | |
150 |
|
150 | |||
151 | def _repr_jpeg_(self): |
|
151 | def _repr_jpeg_(self): | |
152 | pyout = self.metadata['pyout'] or {'data':{}} |
|
152 | pyout = self.metadata['pyout'] or {'data':{}} | |
153 | return pyout['data'].get("image/jpeg") |
|
153 | return pyout['data'].get("image/jpeg") | |
154 |
|
154 | |||
155 | def _repr_svg_(self): |
|
155 | def _repr_svg_(self): | |
156 | pyout = self.metadata['pyout'] or {'data':{}} |
|
156 | pyout = self.metadata['pyout'] or {'data':{}} | |
157 | return pyout['data'].get("image/svg+xml") |
|
157 | return pyout['data'].get("image/svg+xml") | |
158 |
|
158 | |||
159 |
|
159 | |||
160 | class Metadata(dict): |
|
160 | class Metadata(dict): | |
161 | """Subclass of dict for initializing metadata values. |
|
161 | """Subclass of dict for initializing metadata values. | |
162 |
|
162 | |||
163 | Attribute access works on keys. |
|
163 | Attribute access works on keys. | |
164 |
|
164 | |||
165 | These objects have a strict set of keys - errors will raise if you try |
|
165 | These objects have a strict set of keys - errors will raise if you try | |
166 | to add new keys. |
|
166 | to add new keys. | |
167 | """ |
|
167 | """ | |
168 | def __init__(self, *args, **kwargs): |
|
168 | def __init__(self, *args, **kwargs): | |
169 | dict.__init__(self) |
|
169 | dict.__init__(self) | |
170 | md = {'msg_id' : None, |
|
170 | md = {'msg_id' : None, | |
171 | 'submitted' : None, |
|
171 | 'submitted' : None, | |
172 | 'started' : None, |
|
172 | 'started' : None, | |
173 | 'completed' : None, |
|
173 | 'completed' : None, | |
174 | 'received' : None, |
|
174 | 'received' : None, | |
175 | 'engine_uuid' : None, |
|
175 | 'engine_uuid' : None, | |
176 | 'engine_id' : None, |
|
176 | 'engine_id' : None, | |
177 | 'follow' : None, |
|
177 | 'follow' : None, | |
178 | 'after' : None, |
|
178 | 'after' : None, | |
179 | 'status' : None, |
|
179 | 'status' : None, | |
180 |
|
180 | |||
181 | 'pyin' : None, |
|
181 | 'pyin' : None, | |
182 | 'pyout' : None, |
|
182 | 'pyout' : None, | |
183 | 'pyerr' : None, |
|
183 | 'pyerr' : None, | |
184 | 'stdout' : '', |
|
184 | 'stdout' : '', | |
185 | 'stderr' : '', |
|
185 | 'stderr' : '', | |
186 | 'outputs' : [], |
|
186 | 'outputs' : [], | |
187 | 'outputs_ready' : False, |
|
187 | 'outputs_ready' : False, | |
188 | } |
|
188 | } | |
189 | self.update(md) |
|
189 | self.update(md) | |
190 | self.update(dict(*args, **kwargs)) |
|
190 | self.update(dict(*args, **kwargs)) | |
191 |
|
191 | |||
192 | def __getattr__(self, key): |
|
192 | def __getattr__(self, key): | |
193 | """getattr aliased to getitem""" |
|
193 | """getattr aliased to getitem""" | |
194 | if key in self.iterkeys(): |
|
194 | if key in self.iterkeys(): | |
195 | return self[key] |
|
195 | return self[key] | |
196 | else: |
|
196 | else: | |
197 | raise AttributeError(key) |
|
197 | raise AttributeError(key) | |
198 |
|
198 | |||
199 | def __setattr__(self, key, value): |
|
199 | def __setattr__(self, key, value): | |
200 | """setattr aliased to setitem, with strict""" |
|
200 | """setattr aliased to setitem, with strict""" | |
201 | if key in self.iterkeys(): |
|
201 | if key in self.iterkeys(): | |
202 | self[key] = value |
|
202 | self[key] = value | |
203 | else: |
|
203 | else: | |
204 | raise AttributeError(key) |
|
204 | raise AttributeError(key) | |
205 |
|
205 | |||
206 | def __setitem__(self, key, value): |
|
206 | def __setitem__(self, key, value): | |
207 | """strict static key enforcement""" |
|
207 | """strict static key enforcement""" | |
208 | if key in self.iterkeys(): |
|
208 | if key in self.iterkeys(): | |
209 | dict.__setitem__(self, key, value) |
|
209 | dict.__setitem__(self, key, value) | |
210 | else: |
|
210 | else: | |
211 | raise KeyError(key) |
|
211 | raise KeyError(key) | |
212 |
|
212 | |||
213 |
|
213 | |||
214 | class Client(HasTraits): |
|
214 | class Client(HasTraits): | |
215 | """A semi-synchronous client to the IPython ZMQ cluster |
|
215 | """A semi-synchronous client to the IPython ZMQ cluster | |
216 |
|
216 | |||
217 | Parameters |
|
217 | Parameters | |
218 | ---------- |
|
218 | ---------- | |
219 |
|
219 | |||
220 |
url_ |
|
220 | url_file : str/unicode; path to ipcontroller-client.json | |
|
221 | This JSON file should contain all the information needed to connect to a cluster, | |||
|
222 | and is likely the only argument needed. | |||
221 | Connection information for the Hub's registration. If a json connector |
|
223 | Connection information for the Hub's registration. If a json connector | |
222 | file is given, then likely no further configuration is necessary. |
|
224 | file is given, then likely no further configuration is necessary. | |
223 | [Default: use profile] |
|
225 | [Default: use profile] | |
224 | profile : bytes |
|
226 | profile : bytes | |
225 | The name of the Cluster profile to be used to find connector information. |
|
227 | The name of the Cluster profile to be used to find connector information. | |
226 | If run from an IPython application, the default profile will be the same |
|
228 | If run from an IPython application, the default profile will be the same | |
227 | as the running application, otherwise it will be 'default'. |
|
229 | as the running application, otherwise it will be 'default'. | |
228 | context : zmq.Context |
|
230 | context : zmq.Context | |
229 | Pass an existing zmq.Context instance, otherwise the client will create its own. |
|
231 | Pass an existing zmq.Context instance, otherwise the client will create its own. | |
230 | debug : bool |
|
232 | debug : bool | |
231 | flag for lots of message printing for debug purposes |
|
233 | flag for lots of message printing for debug purposes | |
232 | timeout : int/float |
|
234 | timeout : int/float | |
233 | time (in seconds) to wait for connection replies from the Hub |
|
235 | time (in seconds) to wait for connection replies from the Hub | |
234 | [Default: 10] |
|
236 | [Default: 10] | |
235 |
|
237 | |||
236 | #-------------- session related args ---------------- |
|
238 | #-------------- session related args ---------------- | |
237 |
|
239 | |||
238 | config : Config object |
|
240 | config : Config object | |
239 | If specified, this will be relayed to the Session for configuration |
|
241 | If specified, this will be relayed to the Session for configuration | |
240 | username : str |
|
242 | username : str | |
241 | set username for the session object |
|
243 | set username for the session object | |
242 | packer : str (import_string) or callable |
|
|||
243 | Can be either the simple keyword 'json' or 'pickle', or an import_string to a |
|
|||
244 | function to serialize messages. Must support same input as |
|
|||
245 | JSON, and output must be bytes. |
|
|||
246 | You can pass a callable directly as `pack` |
|
|||
247 | unpacker : str (import_string) or callable |
|
|||
248 | The inverse of packer. Only necessary if packer is specified as *not* one |
|
|||
249 | of 'json' or 'pickle'. |
|
|||
250 |
|
244 | |||
251 | #-------------- ssh related args ---------------- |
|
245 | #-------------- ssh related args ---------------- | |
252 | # These are args for configuring the ssh tunnel to be used |
|
246 | # These are args for configuring the ssh tunnel to be used | |
253 | # credentials are used to forward connections over ssh to the Controller |
|
247 | # credentials are used to forward connections over ssh to the Controller | |
254 | # Note that the ip given in `addr` needs to be relative to sshserver |
|
248 | # Note that the ip given in `addr` needs to be relative to sshserver | |
255 | # The most basic case is to leave addr as pointing to localhost (127.0.0.1), |
|
249 | # The most basic case is to leave addr as pointing to localhost (127.0.0.1), | |
256 | # and set sshserver as the same machine the Controller is on. However, |
|
250 | # and set sshserver as the same machine the Controller is on. However, | |
257 | # the only requirement is that sshserver is able to see the Controller |
|
251 | # the only requirement is that sshserver is able to see the Controller | |
258 | # (i.e. is within the same trusted network). |
|
252 | # (i.e. is within the same trusted network). | |
259 |
|
253 | |||
260 | sshserver : str |
|
254 | sshserver : str | |
261 | A string of the form passed to ssh, i.e. 'server.tld' or 'user@server.tld:port' |
|
255 | A string of the form passed to ssh, i.e. 'server.tld' or 'user@server.tld:port' | |
262 | If keyfile or password is specified, and this is not, it will default to |
|
256 | If keyfile or password is specified, and this is not, it will default to | |
263 | the ip given in addr. |
|
257 | the ip given in addr. | |
264 | sshkey : str; path to ssh private key file |
|
258 | sshkey : str; path to ssh private key file | |
265 | This specifies a key to be used in ssh login, default None. |
|
259 | This specifies a key to be used in ssh login, default None. | |
266 | Regular default ssh keys will be used without specifying this argument. |
|
260 | Regular default ssh keys will be used without specifying this argument. | |
267 | password : str |
|
261 | password : str | |
268 | Your ssh password to sshserver. Note that if this is left None, |
|
262 | Your ssh password to sshserver. Note that if this is left None, | |
269 | you will be prompted for it if passwordless key based login is unavailable. |
|
263 | you will be prompted for it if passwordless key based login is unavailable. | |
270 | paramiko : bool |
|
264 | paramiko : bool | |
271 | flag for whether to use paramiko instead of shell ssh for tunneling. |
|
265 | flag for whether to use paramiko instead of shell ssh for tunneling. | |
272 | [default: True on win32, False else] |
|
266 | [default: True on win32, False else] | |
273 |
|
267 | |||
274 | ------- exec authentication args ------- |
|
|||
275 | If even localhost is untrusted, you can have some protection against |
|
|||
276 | unauthorized execution by signing messages with HMAC digests. |
|
|||
277 | Messages are still sent as cleartext, so if someone can snoop your |
|
|||
278 | loopback traffic this will not protect your privacy, but will prevent |
|
|||
279 | unauthorized execution. |
|
|||
280 |
|
||||
281 | exec_key : str |
|
|||
282 | an authentication key or file containing a key |
|
|||
283 | default: None |
|
|||
284 |
|
||||
285 |
|
268 | |||
286 | Attributes |
|
269 | Attributes | |
287 | ---------- |
|
270 | ---------- | |
288 |
|
271 | |||
289 | ids : list of int engine IDs |
|
272 | ids : list of int engine IDs | |
290 | requesting the ids attribute always synchronizes |
|
273 | requesting the ids attribute always synchronizes | |
291 | the registration state. To request ids without synchronization, |
|
274 | the registration state. To request ids without synchronization, | |
292 | use semi-private _ids attributes. |
|
275 | use semi-private _ids attributes. | |
293 |
|
276 | |||
294 | history : list of msg_ids |
|
277 | history : list of msg_ids | |
295 | a list of msg_ids, keeping track of all the execution |
|
278 | a list of msg_ids, keeping track of all the execution | |
296 | messages you have submitted in order. |
|
279 | messages you have submitted in order. | |
297 |
|
280 | |||
298 | outstanding : set of msg_ids |
|
281 | outstanding : set of msg_ids | |
299 | a set of msg_ids that have been submitted, but whose |
|
282 | a set of msg_ids that have been submitted, but whose | |
300 | results have not yet been received. |
|
283 | results have not yet been received. | |
301 |
|
284 | |||
302 | results : dict |
|
285 | results : dict | |
303 | a dict of all our results, keyed by msg_id |
|
286 | a dict of all our results, keyed by msg_id | |
304 |
|
287 | |||
305 | block : bool |
|
288 | block : bool | |
306 | determines default behavior when block not specified |
|
289 | determines default behavior when block not specified | |
307 | in execution methods |
|
290 | in execution methods | |
308 |
|
291 | |||
309 | Methods |
|
292 | Methods | |
310 | ------- |
|
293 | ------- | |
311 |
|
294 | |||
312 | spin |
|
295 | spin | |
313 | flushes incoming results and registration state changes |
|
296 | flushes incoming results and registration state changes | |
314 | control methods spin, and requesting `ids` also ensures up to date |
|
297 | control methods spin, and requesting `ids` also ensures up to date | |
315 |
|
298 | |||
316 | wait |
|
299 | wait | |
317 | wait on one or more msg_ids |
|
300 | wait on one or more msg_ids | |
318 |
|
301 | |||
319 | execution methods |
|
302 | execution methods | |
320 | apply |
|
303 | apply | |
321 | legacy: execute, run |
|
304 | legacy: execute, run | |
322 |
|
305 | |||
323 | data movement |
|
306 | data movement | |
324 | push, pull, scatter, gather |
|
307 | push, pull, scatter, gather | |
325 |
|
308 | |||
326 | query methods |
|
309 | query methods | |
327 | queue_status, get_result, purge, result_status |
|
310 | queue_status, get_result, purge, result_status | |
328 |
|
311 | |||
329 | control methods |
|
312 | control methods | |
330 | abort, shutdown |
|
313 | abort, shutdown | |
331 |
|
314 | |||
332 | """ |
|
315 | """ | |
333 |
|
316 | |||
334 |
|
317 | |||
335 | block = Bool(False) |
|
318 | block = Bool(False) | |
336 | outstanding = Set() |
|
319 | outstanding = Set() | |
337 | results = Instance('collections.defaultdict', (dict,)) |
|
320 | results = Instance('collections.defaultdict', (dict,)) | |
338 | metadata = Instance('collections.defaultdict', (Metadata,)) |
|
321 | metadata = Instance('collections.defaultdict', (Metadata,)) | |
339 | history = List() |
|
322 | history = List() | |
340 | debug = Bool(False) |
|
323 | debug = Bool(False) | |
341 | _spin_thread = Any() |
|
324 | _spin_thread = Any() | |
342 | _stop_spinning = Any() |
|
325 | _stop_spinning = Any() | |
343 |
|
326 | |||
344 | profile=Unicode() |
|
327 | profile=Unicode() | |
345 | def _profile_default(self): |
|
328 | def _profile_default(self): | |
346 | if BaseIPythonApplication.initialized(): |
|
329 | if BaseIPythonApplication.initialized(): | |
347 | # an IPython app *might* be running, try to get its profile |
|
330 | # an IPython app *might* be running, try to get its profile | |
348 | try: |
|
331 | try: | |
349 | return BaseIPythonApplication.instance().profile |
|
332 | return BaseIPythonApplication.instance().profile | |
350 | except (AttributeError, MultipleInstanceError): |
|
333 | except (AttributeError, MultipleInstanceError): | |
351 | # could be a *different* subclass of config.Application, |
|
334 | # could be a *different* subclass of config.Application, | |
352 | # which would raise one of these two errors. |
|
335 | # which would raise one of these two errors. | |
353 | return u'default' |
|
336 | return u'default' | |
354 | else: |
|
337 | else: | |
355 | return u'default' |
|
338 | return u'default' | |
356 |
|
339 | |||
357 |
|
340 | |||
358 | _outstanding_dict = Instance('collections.defaultdict', (set,)) |
|
341 | _outstanding_dict = Instance('collections.defaultdict', (set,)) | |
359 | _ids = List() |
|
342 | _ids = List() | |
360 | _connected=Bool(False) |
|
343 | _connected=Bool(False) | |
361 | _ssh=Bool(False) |
|
344 | _ssh=Bool(False) | |
362 | _context = Instance('zmq.Context') |
|
345 | _context = Instance('zmq.Context') | |
363 | _config = Dict() |
|
346 | _config = Dict() | |
364 | _engines=Instance(util.ReverseDict, (), {}) |
|
347 | _engines=Instance(util.ReverseDict, (), {}) | |
365 | # _hub_socket=Instance('zmq.Socket') |
|
348 | # _hub_socket=Instance('zmq.Socket') | |
366 | _query_socket=Instance('zmq.Socket') |
|
349 | _query_socket=Instance('zmq.Socket') | |
367 | _control_socket=Instance('zmq.Socket') |
|
350 | _control_socket=Instance('zmq.Socket') | |
368 | _iopub_socket=Instance('zmq.Socket') |
|
351 | _iopub_socket=Instance('zmq.Socket') | |
369 | _notification_socket=Instance('zmq.Socket') |
|
352 | _notification_socket=Instance('zmq.Socket') | |
370 | _mux_socket=Instance('zmq.Socket') |
|
353 | _mux_socket=Instance('zmq.Socket') | |
371 | _task_socket=Instance('zmq.Socket') |
|
354 | _task_socket=Instance('zmq.Socket') | |
372 | _task_scheme=Unicode() |
|
355 | _task_scheme=Unicode() | |
373 | _closed = False |
|
356 | _closed = False | |
374 | _ignored_control_replies=Integer(0) |
|
357 | _ignored_control_replies=Integer(0) | |
375 | _ignored_hub_replies=Integer(0) |
|
358 | _ignored_hub_replies=Integer(0) | |
376 |
|
359 | |||
377 | def __new__(self, *args, **kw): |
|
360 | def __new__(self, *args, **kw): | |
378 | # don't raise on positional args |
|
361 | # don't raise on positional args | |
379 | return HasTraits.__new__(self, **kw) |
|
362 | return HasTraits.__new__(self, **kw) | |
380 |
|
363 | |||
381 |
def __init__(self, url_ |
|
364 | def __init__(self, url_file=None, profile=None, profile_dir=None, ipython_dir=None, | |
382 |
context=None, debug=False, |
|
365 | context=None, debug=False, | |
383 | sshserver=None, sshkey=None, password=None, paramiko=None, |
|
366 | sshserver=None, sshkey=None, password=None, paramiko=None, | |
384 | timeout=10, **extra_args |
|
367 | timeout=10, **extra_args | |
385 | ): |
|
368 | ): | |
386 | if profile: |
|
369 | if profile: | |
387 | super(Client, self).__init__(debug=debug, profile=profile) |
|
370 | super(Client, self).__init__(debug=debug, profile=profile) | |
388 | else: |
|
371 | else: | |
389 | super(Client, self).__init__(debug=debug) |
|
372 | super(Client, self).__init__(debug=debug) | |
390 | if context is None: |
|
373 | if context is None: | |
391 | context = zmq.Context.instance() |
|
374 | context = zmq.Context.instance() | |
392 | self._context = context |
|
375 | self._context = context | |
393 | self._stop_spinning = Event() |
|
376 | self._stop_spinning = Event() | |
|
377 | ||||
|
378 | if 'url_or_file' in extra_args: | |||
|
379 | url_file = extra_args['url_or_file'] | |||
|
380 | warnings.warn("url_or_file arg no longer supported, use url_file", DeprecationWarning) | |||
|
381 | ||||
|
382 | if url_file and util.is_url(url_file): | |||
|
383 | raise ValueError("single urls cannot be specified, url-files must be used.") | |||
394 |
|
384 | |||
395 | self._setup_profile_dir(self.profile, profile_dir, ipython_dir) |
|
385 | self._setup_profile_dir(self.profile, profile_dir, ipython_dir) | |
|
386 | ||||
396 | if self._cd is not None: |
|
387 | if self._cd is not None: | |
397 |
if url_ |
|
388 | if url_file is None: | |
398 |
url_ |
|
389 | url_file = pjoin(self._cd.security_dir, 'ipcontroller-client.json') | |
399 |
if url_ |
|
390 | if url_file is None: | |
400 | raise ValueError( |
|
391 | raise ValueError( | |
401 | "I can't find enough information to connect to a hub!" |
|
392 | "I can't find enough information to connect to a hub!" | |
402 |
" Please specify at least one of url_ |
|
393 | " Please specify at least one of url_file or profile." | |
403 | ) |
|
394 | ) | |
404 |
|
395 | |||
405 |
|
|
396 | with open(url_file) as f: | |
406 | # it's not a url, try for a file |
|
397 | cfg = json.load(f) | |
407 | if not os.path.exists(url_or_file): |
|
398 | ||
408 | if self._cd: |
|
399 | self._task_scheme = cfg['task_scheme'] | |
409 | url_or_file = os.path.join(self._cd.security_dir, url_or_file) |
|
|||
410 | if not os.path.exists(url_or_file): |
|
|||
411 | raise IOError("Connection file not found: %r" % url_or_file) |
|
|||
412 | with open(url_or_file) as f: |
|
|||
413 | cfg = json.loads(f.read()) |
|
|||
414 | else: |
|
|||
415 | cfg = {'url':url_or_file} |
|
|||
416 |
|
400 | |||
417 | # sync defaults from args, json: |
|
401 | # sync defaults from args, json: | |
418 | if sshserver: |
|
402 | if sshserver: | |
419 | cfg['ssh'] = sshserver |
|
403 | cfg['ssh'] = sshserver | |
420 | if exec_key: |
|
404 | ||
421 | cfg['exec_key'] = exec_key |
|
|||
422 | exec_key = cfg['exec_key'] |
|
|||
423 | location = cfg.setdefault('location', None) |
|
405 | location = cfg.setdefault('location', None) | |
424 | cfg['url'] = util.disambiguate_url(cfg['url'], location) |
|
406 | for key in ('control', 'task', 'mux', 'notification', 'registration'): | |
425 | url = cfg['url'] |
|
407 | cfg[key] = util.disambiguate_url(cfg[key], location) | |
|
408 | url = cfg['registration'] | |||
426 | proto,addr,port = util.split_url(url) |
|
409 | proto,addr,port = util.split_url(url) | |
427 | if location is not None and addr == '127.0.0.1': |
|
410 | if location is not None and addr == '127.0.0.1': | |
428 | # location specified, and connection is expected to be local |
|
411 | # location specified, and connection is expected to be local | |
429 | if location not in LOCAL_IPS and not sshserver: |
|
412 | if location not in LOCAL_IPS and not sshserver: | |
430 | # load ssh from JSON *only* if the controller is not on |
|
413 | # load ssh from JSON *only* if the controller is not on | |
431 | # this machine |
|
414 | # this machine | |
432 | sshserver=cfg['ssh'] |
|
415 | sshserver=cfg['ssh'] | |
433 | if location not in LOCAL_IPS and not sshserver: |
|
416 | if location not in LOCAL_IPS and not sshserver: | |
434 | # warn if no ssh specified, but SSH is probably needed |
|
417 | # warn if no ssh specified, but SSH is probably needed | |
435 | # This is only a warning, because the most likely cause |
|
418 | # This is only a warning, because the most likely cause | |
436 | # is a local Controller on a laptop whose IP is dynamic |
|
419 | # is a local Controller on a laptop whose IP is dynamic | |
437 | warnings.warn(""" |
|
420 | warnings.warn(""" | |
438 | Controller appears to be listening on localhost, but not on this machine. |
|
421 | Controller appears to be listening on localhost, but not on this machine. | |
439 | If this is true, you should specify Client(...,sshserver='you@%s') |
|
422 | If this is true, you should specify Client(...,sshserver='you@%s') | |
440 | or instruct your controller to listen on an external IP."""%location, |
|
423 | or instruct your controller to listen on an external IP."""%location, | |
441 | RuntimeWarning) |
|
424 | RuntimeWarning) | |
442 | elif not sshserver: |
|
425 | elif not sshserver: | |
443 | # otherwise sync with cfg |
|
426 | # otherwise sync with cfg | |
444 | sshserver = cfg['ssh'] |
|
427 | sshserver = cfg['ssh'] | |
445 |
|
428 | |||
446 | self._config = cfg |
|
429 | self._config = cfg | |
447 |
|
430 | |||
448 | self._ssh = bool(sshserver or sshkey or password) |
|
431 | self._ssh = bool(sshserver or sshkey or password) | |
449 | if self._ssh and sshserver is None: |
|
432 | if self._ssh and sshserver is None: | |
450 | # default to ssh via localhost |
|
433 | # default to ssh via localhost | |
451 | sshserver = url.split('://')[1].split(':')[0] |
|
434 | sshserver = url.split('://')[1].split(':')[0] | |
452 | if self._ssh and password is None: |
|
435 | if self._ssh and password is None: | |
453 | if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko): |
|
436 | if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko): | |
454 | password=False |
|
437 | password=False | |
455 | else: |
|
438 | else: | |
456 | password = getpass("SSH Password for %s: "%sshserver) |
|
439 | password = getpass("SSH Password for %s: "%sshserver) | |
457 | ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko) |
|
440 | ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko) | |
458 |
|
441 | |||
459 | # configure and construct the session |
|
442 | # configure and construct the session | |
460 | if exec_key is not None: |
|
443 | extra_args['packer'] = cfg['pack'] | |
461 | if os.path.isfile(exec_key): |
|
444 | extra_args['unpacker'] = cfg['unpack'] | |
462 |
|
|
445 | extra_args['key'] = cfg['exec_key'] | |
463 |
|
|
446 | ||
464 | exec_key = cast_bytes(exec_key) |
|
|||
465 | extra_args['key'] = exec_key |
|
|||
466 | self.session = Session(**extra_args) |
|
447 | self.session = Session(**extra_args) | |
467 |
|
448 | |||
468 | self._query_socket = self._context.socket(zmq.DEALER) |
|
449 | self._query_socket = self._context.socket(zmq.DEALER) | |
469 |
|
450 | |||
470 | if self._ssh: |
|
451 | if self._ssh: | |
471 | tunnel.tunnel_connection(self._query_socket, url, sshserver, **ssh_kwargs) |
|
452 | tunnel.tunnel_connection(self._query_socket, url, sshserver, **ssh_kwargs) | |
472 | else: |
|
453 | else: | |
473 | self._query_socket.connect(url) |
|
454 | self._query_socket.connect(url) | |
474 |
|
455 | |||
475 | self.session.debug = self.debug |
|
456 | self.session.debug = self.debug | |
476 |
|
457 | |||
477 | self._notification_handlers = {'registration_notification' : self._register_engine, |
|
458 | self._notification_handlers = {'registration_notification' : self._register_engine, | |
478 | 'unregistration_notification' : self._unregister_engine, |
|
459 | 'unregistration_notification' : self._unregister_engine, | |
479 | 'shutdown_notification' : lambda msg: self.close(), |
|
460 | 'shutdown_notification' : lambda msg: self.close(), | |
480 | } |
|
461 | } | |
481 | self._queue_handlers = {'execute_reply' : self._handle_execute_reply, |
|
462 | self._queue_handlers = {'execute_reply' : self._handle_execute_reply, | |
482 | 'apply_reply' : self._handle_apply_reply} |
|
463 | 'apply_reply' : self._handle_apply_reply} | |
483 | self._connect(sshserver, ssh_kwargs, timeout) |
|
464 | self._connect(sshserver, ssh_kwargs, timeout) | |
484 |
|
465 | |||
485 | # last step: setup magics, if we are in IPython: |
|
466 | # last step: setup magics, if we are in IPython: | |
486 |
|
467 | |||
487 | try: |
|
468 | try: | |
488 | ip = get_ipython() |
|
469 | ip = get_ipython() | |
489 | except NameError: |
|
470 | except NameError: | |
490 | return |
|
471 | return | |
491 | else: |
|
472 | else: | |
492 | if 'px' not in ip.magics_manager.magics: |
|
473 | if 'px' not in ip.magics_manager.magics: | |
493 | # in IPython but we are the first Client. |
|
474 | # in IPython but we are the first Client. | |
494 | # activate a default view for parallel magics. |
|
475 | # activate a default view for parallel magics. | |
495 | self.activate() |
|
476 | self.activate() | |
496 |
|
477 | |||
497 | def __del__(self): |
|
478 | def __del__(self): | |
498 | """cleanup sockets, but _not_ context.""" |
|
479 | """cleanup sockets, but _not_ context.""" | |
499 | self.close() |
|
480 | self.close() | |
500 |
|
481 | |||
501 | def _setup_profile_dir(self, profile, profile_dir, ipython_dir): |
|
482 | def _setup_profile_dir(self, profile, profile_dir, ipython_dir): | |
502 | if ipython_dir is None: |
|
483 | if ipython_dir is None: | |
503 | ipython_dir = get_ipython_dir() |
|
484 | ipython_dir = get_ipython_dir() | |
504 | if profile_dir is not None: |
|
485 | if profile_dir is not None: | |
505 | try: |
|
486 | try: | |
506 | self._cd = ProfileDir.find_profile_dir(profile_dir) |
|
487 | self._cd = ProfileDir.find_profile_dir(profile_dir) | |
507 | return |
|
488 | return | |
508 | except ProfileDirError: |
|
489 | except ProfileDirError: | |
509 | pass |
|
490 | pass | |
510 | elif profile is not None: |
|
491 | elif profile is not None: | |
511 | try: |
|
492 | try: | |
512 | self._cd = ProfileDir.find_profile_dir_by_name( |
|
493 | self._cd = ProfileDir.find_profile_dir_by_name( | |
513 | ipython_dir, profile) |
|
494 | ipython_dir, profile) | |
514 | return |
|
495 | return | |
515 | except ProfileDirError: |
|
496 | except ProfileDirError: | |
516 | pass |
|
497 | pass | |
517 | self._cd = None |
|
498 | self._cd = None | |
518 |
|
499 | |||
519 | def _update_engines(self, engines): |
|
500 | def _update_engines(self, engines): | |
520 | """Update our engines dict and _ids from a dict of the form: {id:uuid}.""" |
|
501 | """Update our engines dict and _ids from a dict of the form: {id:uuid}.""" | |
521 | for k,v in engines.iteritems(): |
|
502 | for k,v in engines.iteritems(): | |
522 | eid = int(k) |
|
503 | eid = int(k) | |
523 | self._engines[eid] = v |
|
504 | self._engines[eid] = v | |
524 | self._ids.append(eid) |
|
505 | self._ids.append(eid) | |
525 | self._ids = sorted(self._ids) |
|
506 | self._ids = sorted(self._ids) | |
526 | if sorted(self._engines.keys()) != range(len(self._engines)) and \ |
|
507 | if sorted(self._engines.keys()) != range(len(self._engines)) and \ | |
527 | self._task_scheme == 'pure' and self._task_socket: |
|
508 | self._task_scheme == 'pure' and self._task_socket: | |
528 | self._stop_scheduling_tasks() |
|
509 | self._stop_scheduling_tasks() | |
529 |
|
510 | |||
530 | def _stop_scheduling_tasks(self): |
|
511 | def _stop_scheduling_tasks(self): | |
531 | """Stop scheduling tasks because an engine has been unregistered |
|
512 | """Stop scheduling tasks because an engine has been unregistered | |
532 | from a pure ZMQ scheduler. |
|
513 | from a pure ZMQ scheduler. | |
533 | """ |
|
514 | """ | |
534 | self._task_socket.close() |
|
515 | self._task_socket.close() | |
535 | self._task_socket = None |
|
516 | self._task_socket = None | |
536 | msg = "An engine has been unregistered, and we are using pure " +\ |
|
517 | msg = "An engine has been unregistered, and we are using pure " +\ | |
537 | "ZMQ task scheduling. Task farming will be disabled." |
|
518 | "ZMQ task scheduling. Task farming will be disabled." | |
538 | if self.outstanding: |
|
519 | if self.outstanding: | |
539 | msg += " If you were running tasks when this happened, " +\ |
|
520 | msg += " If you were running tasks when this happened, " +\ | |
540 | "some `outstanding` msg_ids may never resolve." |
|
521 | "some `outstanding` msg_ids may never resolve." | |
541 | warnings.warn(msg, RuntimeWarning) |
|
522 | warnings.warn(msg, RuntimeWarning) | |
542 |
|
523 | |||
543 | def _build_targets(self, targets): |
|
524 | def _build_targets(self, targets): | |
544 | """Turn valid target IDs or 'all' into two lists: |
|
525 | """Turn valid target IDs or 'all' into two lists: | |
545 | (int_ids, uuids). |
|
526 | (int_ids, uuids). | |
546 | """ |
|
527 | """ | |
547 | if not self._ids: |
|
528 | if not self._ids: | |
548 | # flush notification socket if no engines yet, just in case |
|
529 | # flush notification socket if no engines yet, just in case | |
549 | if not self.ids: |
|
530 | if not self.ids: | |
550 | raise error.NoEnginesRegistered("Can't build targets without any engines") |
|
531 | raise error.NoEnginesRegistered("Can't build targets without any engines") | |
551 |
|
532 | |||
552 | if targets is None: |
|
533 | if targets is None: | |
553 | targets = self._ids |
|
534 | targets = self._ids | |
554 | elif isinstance(targets, basestring): |
|
535 | elif isinstance(targets, basestring): | |
555 | if targets.lower() == 'all': |
|
536 | if targets.lower() == 'all': | |
556 | targets = self._ids |
|
537 | targets = self._ids | |
557 | else: |
|
538 | else: | |
558 | raise TypeError("%r not valid str target, must be 'all'"%(targets)) |
|
539 | raise TypeError("%r not valid str target, must be 'all'"%(targets)) | |
559 | elif isinstance(targets, int): |
|
540 | elif isinstance(targets, int): | |
560 | if targets < 0: |
|
541 | if targets < 0: | |
561 | targets = self.ids[targets] |
|
542 | targets = self.ids[targets] | |
562 | if targets not in self._ids: |
|
543 | if targets not in self._ids: | |
563 | raise IndexError("No such engine: %i"%targets) |
|
544 | raise IndexError("No such engine: %i"%targets) | |
564 | targets = [targets] |
|
545 | targets = [targets] | |
565 |
|
546 | |||
566 | if isinstance(targets, slice): |
|
547 | if isinstance(targets, slice): | |
567 | indices = range(len(self._ids))[targets] |
|
548 | indices = range(len(self._ids))[targets] | |
568 | ids = self.ids |
|
549 | ids = self.ids | |
569 | targets = [ ids[i] for i in indices ] |
|
550 | targets = [ ids[i] for i in indices ] | |
570 |
|
551 | |||
571 | if not isinstance(targets, (tuple, list, xrange)): |
|
552 | if not isinstance(targets, (tuple, list, xrange)): | |
572 | raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets))) |
|
553 | raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets))) | |
573 |
|
554 | |||
574 | return [cast_bytes(self._engines[t]) for t in targets], list(targets) |
|
555 | return [cast_bytes(self._engines[t]) for t in targets], list(targets) | |
575 |
|
556 | |||
576 | def _connect(self, sshserver, ssh_kwargs, timeout): |
|
557 | def _connect(self, sshserver, ssh_kwargs, timeout): | |
577 | """setup all our socket connections to the cluster. This is called from |
|
558 | """setup all our socket connections to the cluster. This is called from | |
578 | __init__.""" |
|
559 | __init__.""" | |
579 |
|
560 | |||
580 | # Maybe allow reconnecting? |
|
561 | # Maybe allow reconnecting? | |
581 | if self._connected: |
|
562 | if self._connected: | |
582 | return |
|
563 | return | |
583 | self._connected=True |
|
564 | self._connected=True | |
584 |
|
565 | |||
585 | def connect_socket(s, url): |
|
566 | def connect_socket(s, url): | |
586 | url = util.disambiguate_url(url, self._config['location']) |
|
567 | # url = util.disambiguate_url(url, self._config['location']) | |
587 | if self._ssh: |
|
568 | if self._ssh: | |
588 | return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs) |
|
569 | return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs) | |
589 | else: |
|
570 | else: | |
590 | return s.connect(url) |
|
571 | return s.connect(url) | |
591 |
|
572 | |||
592 | self.session.send(self._query_socket, 'connection_request') |
|
573 | self.session.send(self._query_socket, 'connection_request') | |
593 | # use Poller because zmq.select has wrong units in pyzmq 2.1.7 |
|
574 | # use Poller because zmq.select has wrong units in pyzmq 2.1.7 | |
594 | poller = zmq.Poller() |
|
575 | poller = zmq.Poller() | |
595 | poller.register(self._query_socket, zmq.POLLIN) |
|
576 | poller.register(self._query_socket, zmq.POLLIN) | |
596 | # poll expects milliseconds, timeout is seconds |
|
577 | # poll expects milliseconds, timeout is seconds | |
597 | evts = poller.poll(timeout*1000) |
|
578 | evts = poller.poll(timeout*1000) | |
598 | if not evts: |
|
579 | if not evts: | |
599 | raise error.TimeoutError("Hub connection request timed out") |
|
580 | raise error.TimeoutError("Hub connection request timed out") | |
600 | idents,msg = self.session.recv(self._query_socket,mode=0) |
|
581 | idents,msg = self.session.recv(self._query_socket,mode=0) | |
601 | if self.debug: |
|
582 | if self.debug: | |
602 | pprint(msg) |
|
583 | pprint(msg) | |
603 | msg = Message(msg) |
|
584 | content = msg['content'] | |
604 | content = msg.content |
|
585 | # self._config['registration'] = dict(content) | |
605 | self._config['registration'] = dict(content) |
|
586 | cfg = self._config | |
606 |
if content |
|
587 | if content['status'] == 'ok': | |
607 | ident = self.session.bsession |
|
588 | self._mux_socket = self._context.socket(zmq.DEALER) | |
608 | if content.mux: |
|
589 | connect_socket(self._mux_socket, cfg['mux']) | |
609 | self._mux_socket = self._context.socket(zmq.DEALER) |
|
590 | ||
610 | connect_socket(self._mux_socket, content.mux) |
|
591 | self._task_socket = self._context.socket(zmq.DEALER) | |
611 | if content.task: |
|
592 | connect_socket(self._task_socket, cfg['task']) | |
612 | self._task_scheme, task_addr = content.task |
|
593 | ||
613 |
|
|
594 | self._notification_socket = self._context.socket(zmq.SUB) | |
614 | connect_socket(self._task_socket, task_addr) |
|
595 | self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'') | |
615 | if content.notification: |
|
596 | connect_socket(self._notification_socket, cfg['notification']) | |
616 | self._notification_socket = self._context.socket(zmq.SUB) |
|
597 | ||
617 | connect_socket(self._notification_socket, content.notification) |
|
598 | self._control_socket = self._context.socket(zmq.DEALER) | |
618 | self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'') |
|
599 | connect_socket(self._control_socket, cfg['control']) | |
619 | if content.control: |
|
600 | ||
620 |
|
|
601 | self._iopub_socket = self._context.socket(zmq.SUB) | |
621 | connect_socket(self._control_socket, content.control) |
|
602 | self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'') | |
622 | if content.iopub: |
|
603 | connect_socket(self._iopub_socket, cfg['iopub']) | |
623 | self._iopub_socket = self._context.socket(zmq.SUB) |
|
604 | ||
624 | self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'') |
|
605 | self._update_engines(dict(content['engines'])) | |
625 | connect_socket(self._iopub_socket, content.iopub) |
|
|||
626 | self._update_engines(dict(content.engines)) |
|
|||
627 | else: |
|
606 | else: | |
628 | self._connected = False |
|
607 | self._connected = False | |
629 | raise Exception("Failed to connect!") |
|
608 | raise Exception("Failed to connect!") | |
630 |
|
609 | |||
631 | #-------------------------------------------------------------------------- |
|
610 | #-------------------------------------------------------------------------- | |
632 | # handlers and callbacks for incoming messages |
|
611 | # handlers and callbacks for incoming messages | |
633 | #-------------------------------------------------------------------------- |
|
612 | #-------------------------------------------------------------------------- | |
634 |
|
613 | |||
635 | def _unwrap_exception(self, content): |
|
614 | def _unwrap_exception(self, content): | |
636 | """unwrap exception, and remap engine_id to int.""" |
|
615 | """unwrap exception, and remap engine_id to int.""" | |
637 | e = error.unwrap_exception(content) |
|
616 | e = error.unwrap_exception(content) | |
638 | # print e.traceback |
|
617 | # print e.traceback | |
639 | if e.engine_info: |
|
618 | if e.engine_info: | |
640 | e_uuid = e.engine_info['engine_uuid'] |
|
619 | e_uuid = e.engine_info['engine_uuid'] | |
641 | eid = self._engines[e_uuid] |
|
620 | eid = self._engines[e_uuid] | |
642 | e.engine_info['engine_id'] = eid |
|
621 | e.engine_info['engine_id'] = eid | |
643 | return e |
|
622 | return e | |
644 |
|
623 | |||
645 | def _extract_metadata(self, header, parent, content): |
|
624 | def _extract_metadata(self, header, parent, content): | |
646 | md = {'msg_id' : parent['msg_id'], |
|
625 | md = {'msg_id' : parent['msg_id'], | |
647 | 'received' : datetime.now(), |
|
626 | 'received' : datetime.now(), | |
648 | 'engine_uuid' : header.get('engine', None), |
|
627 | 'engine_uuid' : header.get('engine', None), | |
649 | 'follow' : parent.get('follow', []), |
|
628 | 'follow' : parent.get('follow', []), | |
650 | 'after' : parent.get('after', []), |
|
629 | 'after' : parent.get('after', []), | |
651 | 'status' : content['status'], |
|
630 | 'status' : content['status'], | |
652 | } |
|
631 | } | |
653 |
|
632 | |||
654 | if md['engine_uuid'] is not None: |
|
633 | if md['engine_uuid'] is not None: | |
655 | md['engine_id'] = self._engines.get(md['engine_uuid'], None) |
|
634 | md['engine_id'] = self._engines.get(md['engine_uuid'], None) | |
656 |
|
635 | |||
657 | if 'date' in parent: |
|
636 | if 'date' in parent: | |
658 | md['submitted'] = parent['date'] |
|
637 | md['submitted'] = parent['date'] | |
659 | if 'started' in header: |
|
638 | if 'started' in header: | |
660 | md['started'] = header['started'] |
|
639 | md['started'] = header['started'] | |
661 | if 'date' in header: |
|
640 | if 'date' in header: | |
662 | md['completed'] = header['date'] |
|
641 | md['completed'] = header['date'] | |
663 | return md |
|
642 | return md | |
664 |
|
643 | |||
665 | def _register_engine(self, msg): |
|
644 | def _register_engine(self, msg): | |
666 | """Register a new engine, and update our connection info.""" |
|
645 | """Register a new engine, and update our connection info.""" | |
667 | content = msg['content'] |
|
646 | content = msg['content'] | |
668 | eid = content['id'] |
|
647 | eid = content['id'] | |
669 | d = {eid : content['queue']} |
|
648 | d = {eid : content['queue']} | |
670 | self._update_engines(d) |
|
649 | self._update_engines(d) | |
671 |
|
650 | |||
672 | def _unregister_engine(self, msg): |
|
651 | def _unregister_engine(self, msg): | |
673 | """Unregister an engine that has died.""" |
|
652 | """Unregister an engine that has died.""" | |
674 | content = msg['content'] |
|
653 | content = msg['content'] | |
675 | eid = int(content['id']) |
|
654 | eid = int(content['id']) | |
676 | if eid in self._ids: |
|
655 | if eid in self._ids: | |
677 | self._ids.remove(eid) |
|
656 | self._ids.remove(eid) | |
678 | uuid = self._engines.pop(eid) |
|
657 | uuid = self._engines.pop(eid) | |
679 |
|
658 | |||
680 | self._handle_stranded_msgs(eid, uuid) |
|
659 | self._handle_stranded_msgs(eid, uuid) | |
681 |
|
660 | |||
682 | if self._task_socket and self._task_scheme == 'pure': |
|
661 | if self._task_socket and self._task_scheme == 'pure': | |
683 | self._stop_scheduling_tasks() |
|
662 | self._stop_scheduling_tasks() | |
684 |
|
663 | |||
685 | def _handle_stranded_msgs(self, eid, uuid): |
|
664 | def _handle_stranded_msgs(self, eid, uuid): | |
686 | """Handle messages known to be on an engine when the engine unregisters. |
|
665 | """Handle messages known to be on an engine when the engine unregisters. | |
687 |
|
666 | |||
688 | It is possible that this will fire prematurely - that is, an engine will |
|
667 | It is possible that this will fire prematurely - that is, an engine will | |
689 | go down after completing a result, and the client will be notified |
|
668 | go down after completing a result, and the client will be notified | |
690 | of the unregistration and later receive the successful result. |
|
669 | of the unregistration and later receive the successful result. | |
691 | """ |
|
670 | """ | |
692 |
|
671 | |||
693 | outstanding = self._outstanding_dict[uuid] |
|
672 | outstanding = self._outstanding_dict[uuid] | |
694 |
|
673 | |||
695 | for msg_id in list(outstanding): |
|
674 | for msg_id in list(outstanding): | |
696 | if msg_id in self.results: |
|
675 | if msg_id in self.results: | |
697 | # we already |
|
676 | # we already | |
698 | continue |
|
677 | continue | |
699 | try: |
|
678 | try: | |
700 | raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id)) |
|
679 | raise error.EngineError("Engine %r died while running task %r"%(eid, msg_id)) | |
701 | except: |
|
680 | except: | |
702 | content = error.wrap_exception() |
|
681 | content = error.wrap_exception() | |
703 | # build a fake message: |
|
682 | # build a fake message: | |
704 | parent = {} |
|
683 | parent = {} | |
705 | header = {} |
|
684 | header = {} | |
706 | parent['msg_id'] = msg_id |
|
685 | parent['msg_id'] = msg_id | |
707 | header['engine'] = uuid |
|
686 | header['engine'] = uuid | |
708 | header['date'] = datetime.now() |
|
687 | header['date'] = datetime.now() | |
709 | msg = dict(parent_header=parent, header=header, content=content) |
|
688 | msg = dict(parent_header=parent, header=header, content=content) | |
710 | self._handle_apply_reply(msg) |
|
689 | self._handle_apply_reply(msg) | |
711 |
|
690 | |||
712 | def _handle_execute_reply(self, msg): |
|
691 | def _handle_execute_reply(self, msg): | |
713 | """Save the reply to an execute_request into our results. |
|
692 | """Save the reply to an execute_request into our results. | |
714 |
|
693 | |||
715 | execute messages are never actually used. apply is used instead. |
|
694 | execute messages are never actually used. apply is used instead. | |
716 | """ |
|
695 | """ | |
717 |
|
696 | |||
718 | parent = msg['parent_header'] |
|
697 | parent = msg['parent_header'] | |
719 | msg_id = parent['msg_id'] |
|
698 | msg_id = parent['msg_id'] | |
720 | if msg_id not in self.outstanding: |
|
699 | if msg_id not in self.outstanding: | |
721 | if msg_id in self.history: |
|
700 | if msg_id in self.history: | |
722 | print ("got stale result: %s"%msg_id) |
|
701 | print ("got stale result: %s"%msg_id) | |
723 | else: |
|
702 | else: | |
724 | print ("got unknown result: %s"%msg_id) |
|
703 | print ("got unknown result: %s"%msg_id) | |
725 | else: |
|
704 | else: | |
726 | self.outstanding.remove(msg_id) |
|
705 | self.outstanding.remove(msg_id) | |
727 |
|
706 | |||
728 | content = msg['content'] |
|
707 | content = msg['content'] | |
729 | header = msg['header'] |
|
708 | header = msg['header'] | |
730 |
|
709 | |||
731 | # construct metadata: |
|
710 | # construct metadata: | |
732 | md = self.metadata[msg_id] |
|
711 | md = self.metadata[msg_id] | |
733 | md.update(self._extract_metadata(header, parent, content)) |
|
712 | md.update(self._extract_metadata(header, parent, content)) | |
734 | # is this redundant? |
|
713 | # is this redundant? | |
735 | self.metadata[msg_id] = md |
|
714 | self.metadata[msg_id] = md | |
736 |
|
715 | |||
737 | e_outstanding = self._outstanding_dict[md['engine_uuid']] |
|
716 | e_outstanding = self._outstanding_dict[md['engine_uuid']] | |
738 | if msg_id in e_outstanding: |
|
717 | if msg_id in e_outstanding: | |
739 | e_outstanding.remove(msg_id) |
|
718 | e_outstanding.remove(msg_id) | |
740 |
|
719 | |||
741 | # construct result: |
|
720 | # construct result: | |
742 | if content['status'] == 'ok': |
|
721 | if content['status'] == 'ok': | |
743 | self.results[msg_id] = ExecuteReply(msg_id, content, md) |
|
722 | self.results[msg_id] = ExecuteReply(msg_id, content, md) | |
744 | elif content['status'] == 'aborted': |
|
723 | elif content['status'] == 'aborted': | |
745 | self.results[msg_id] = error.TaskAborted(msg_id) |
|
724 | self.results[msg_id] = error.TaskAborted(msg_id) | |
746 | elif content['status'] == 'resubmitted': |
|
725 | elif content['status'] == 'resubmitted': | |
747 | # TODO: handle resubmission |
|
726 | # TODO: handle resubmission | |
748 | pass |
|
727 | pass | |
749 | else: |
|
728 | else: | |
750 | self.results[msg_id] = self._unwrap_exception(content) |
|
729 | self.results[msg_id] = self._unwrap_exception(content) | |
751 |
|
730 | |||
752 | def _handle_apply_reply(self, msg): |
|
731 | def _handle_apply_reply(self, msg): | |
753 | """Save the reply to an apply_request into our results.""" |
|
732 | """Save the reply to an apply_request into our results.""" | |
754 | parent = msg['parent_header'] |
|
733 | parent = msg['parent_header'] | |
755 | msg_id = parent['msg_id'] |
|
734 | msg_id = parent['msg_id'] | |
756 | if msg_id not in self.outstanding: |
|
735 | if msg_id not in self.outstanding: | |
757 | if msg_id in self.history: |
|
736 | if msg_id in self.history: | |
758 | print ("got stale result: %s"%msg_id) |
|
737 | print ("got stale result: %s"%msg_id) | |
759 | print self.results[msg_id] |
|
738 | print self.results[msg_id] | |
760 | print msg |
|
739 | print msg | |
761 | else: |
|
740 | else: | |
762 | print ("got unknown result: %s"%msg_id) |
|
741 | print ("got unknown result: %s"%msg_id) | |
763 | else: |
|
742 | else: | |
764 | self.outstanding.remove(msg_id) |
|
743 | self.outstanding.remove(msg_id) | |
765 | content = msg['content'] |
|
744 | content = msg['content'] | |
766 | header = msg['header'] |
|
745 | header = msg['header'] | |
767 |
|
746 | |||
768 | # construct metadata: |
|
747 | # construct metadata: | |
769 | md = self.metadata[msg_id] |
|
748 | md = self.metadata[msg_id] | |
770 | md.update(self._extract_metadata(header, parent, content)) |
|
749 | md.update(self._extract_metadata(header, parent, content)) | |
771 | # is this redundant? |
|
750 | # is this redundant? | |
772 | self.metadata[msg_id] = md |
|
751 | self.metadata[msg_id] = md | |
773 |
|
752 | |||
774 | e_outstanding = self._outstanding_dict[md['engine_uuid']] |
|
753 | e_outstanding = self._outstanding_dict[md['engine_uuid']] | |
775 | if msg_id in e_outstanding: |
|
754 | if msg_id in e_outstanding: | |
776 | e_outstanding.remove(msg_id) |
|
755 | e_outstanding.remove(msg_id) | |
777 |
|
756 | |||
778 | # construct result: |
|
757 | # construct result: | |
779 | if content['status'] == 'ok': |
|
758 | if content['status'] == 'ok': | |
780 | self.results[msg_id] = util.unserialize_object(msg['buffers'])[0] |
|
759 | self.results[msg_id] = util.unserialize_object(msg['buffers'])[0] | |
781 | elif content['status'] == 'aborted': |
|
760 | elif content['status'] == 'aborted': | |
782 | self.results[msg_id] = error.TaskAborted(msg_id) |
|
761 | self.results[msg_id] = error.TaskAborted(msg_id) | |
783 | elif content['status'] == 'resubmitted': |
|
762 | elif content['status'] == 'resubmitted': | |
784 | # TODO: handle resubmission |
|
763 | # TODO: handle resubmission | |
785 | pass |
|
764 | pass | |
786 | else: |
|
765 | else: | |
787 | self.results[msg_id] = self._unwrap_exception(content) |
|
766 | self.results[msg_id] = self._unwrap_exception(content) | |
788 |
|
767 | |||
789 | def _flush_notifications(self): |
|
768 | def _flush_notifications(self): | |
790 | """Flush notifications of engine registrations waiting |
|
769 | """Flush notifications of engine registrations waiting | |
791 | in ZMQ queue.""" |
|
770 | in ZMQ queue.""" | |
792 | idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK) |
|
771 | idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK) | |
793 | while msg is not None: |
|
772 | while msg is not None: | |
794 | if self.debug: |
|
773 | if self.debug: | |
795 | pprint(msg) |
|
774 | pprint(msg) | |
796 | msg_type = msg['header']['msg_type'] |
|
775 | msg_type = msg['header']['msg_type'] | |
797 | handler = self._notification_handlers.get(msg_type, None) |
|
776 | handler = self._notification_handlers.get(msg_type, None) | |
798 | if handler is None: |
|
777 | if handler is None: | |
799 | raise Exception("Unhandled message type: %s"%msg.msg_type) |
|
778 | raise Exception("Unhandled message type: %s"%msg.msg_type) | |
800 | else: |
|
779 | else: | |
801 | handler(msg) |
|
780 | handler(msg) | |
802 | idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK) |
|
781 | idents,msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK) | |
803 |
|
782 | |||
804 | def _flush_results(self, sock): |
|
783 | def _flush_results(self, sock): | |
805 | """Flush task or queue results waiting in ZMQ queue.""" |
|
784 | """Flush task or queue results waiting in ZMQ queue.""" | |
806 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
785 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
807 | while msg is not None: |
|
786 | while msg is not None: | |
808 | if self.debug: |
|
787 | if self.debug: | |
809 | pprint(msg) |
|
788 | pprint(msg) | |
810 | msg_type = msg['header']['msg_type'] |
|
789 | msg_type = msg['header']['msg_type'] | |
811 | handler = self._queue_handlers.get(msg_type, None) |
|
790 | handler = self._queue_handlers.get(msg_type, None) | |
812 | if handler is None: |
|
791 | if handler is None: | |
813 | raise Exception("Unhandled message type: %s"%msg.msg_type) |
|
792 | raise Exception("Unhandled message type: %s"%msg.msg_type) | |
814 | else: |
|
793 | else: | |
815 | handler(msg) |
|
794 | handler(msg) | |
816 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
795 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
817 |
|
796 | |||
818 | def _flush_control(self, sock): |
|
797 | def _flush_control(self, sock): | |
819 | """Flush replies from the control channel waiting |
|
798 | """Flush replies from the control channel waiting | |
820 | in the ZMQ queue. |
|
799 | in the ZMQ queue. | |
821 |
|
800 | |||
822 | Currently: ignore them.""" |
|
801 | Currently: ignore them.""" | |
823 | if self._ignored_control_replies <= 0: |
|
802 | if self._ignored_control_replies <= 0: | |
824 | return |
|
803 | return | |
825 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
804 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
826 | while msg is not None: |
|
805 | while msg is not None: | |
827 | self._ignored_control_replies -= 1 |
|
806 | self._ignored_control_replies -= 1 | |
828 | if self.debug: |
|
807 | if self.debug: | |
829 | pprint(msg) |
|
808 | pprint(msg) | |
830 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
809 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
831 |
|
810 | |||
832 | def _flush_ignored_control(self): |
|
811 | def _flush_ignored_control(self): | |
833 | """flush ignored control replies""" |
|
812 | """flush ignored control replies""" | |
834 | while self._ignored_control_replies > 0: |
|
813 | while self._ignored_control_replies > 0: | |
835 | self.session.recv(self._control_socket) |
|
814 | self.session.recv(self._control_socket) | |
836 | self._ignored_control_replies -= 1 |
|
815 | self._ignored_control_replies -= 1 | |
837 |
|
816 | |||
838 | def _flush_ignored_hub_replies(self): |
|
817 | def _flush_ignored_hub_replies(self): | |
839 | ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK) |
|
818 | ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK) | |
840 | while msg is not None: |
|
819 | while msg is not None: | |
841 | ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK) |
|
820 | ident,msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK) | |
842 |
|
821 | |||
843 | def _flush_iopub(self, sock): |
|
822 | def _flush_iopub(self, sock): | |
844 | """Flush replies from the iopub channel waiting |
|
823 | """Flush replies from the iopub channel waiting | |
845 | in the ZMQ queue. |
|
824 | in the ZMQ queue. | |
846 | """ |
|
825 | """ | |
847 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
826 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
848 | while msg is not None: |
|
827 | while msg is not None: | |
849 | if self.debug: |
|
828 | if self.debug: | |
850 | pprint(msg) |
|
829 | pprint(msg) | |
851 | parent = msg['parent_header'] |
|
830 | parent = msg['parent_header'] | |
852 | # ignore IOPub messages with no parent. |
|
831 | # ignore IOPub messages with no parent. | |
853 | # Caused by print statements or warnings from before the first execution. |
|
832 | # Caused by print statements or warnings from before the first execution. | |
854 | if not parent: |
|
833 | if not parent: | |
855 | continue |
|
834 | continue | |
856 | msg_id = parent['msg_id'] |
|
835 | msg_id = parent['msg_id'] | |
857 | content = msg['content'] |
|
836 | content = msg['content'] | |
858 | header = msg['header'] |
|
837 | header = msg['header'] | |
859 | msg_type = msg['header']['msg_type'] |
|
838 | msg_type = msg['header']['msg_type'] | |
860 |
|
839 | |||
861 | # init metadata: |
|
840 | # init metadata: | |
862 | md = self.metadata[msg_id] |
|
841 | md = self.metadata[msg_id] | |
863 |
|
842 | |||
864 | if msg_type == 'stream': |
|
843 | if msg_type == 'stream': | |
865 | name = content['name'] |
|
844 | name = content['name'] | |
866 | s = md[name] or '' |
|
845 | s = md[name] or '' | |
867 | md[name] = s + content['data'] |
|
846 | md[name] = s + content['data'] | |
868 | elif msg_type == 'pyerr': |
|
847 | elif msg_type == 'pyerr': | |
869 | md.update({'pyerr' : self._unwrap_exception(content)}) |
|
848 | md.update({'pyerr' : self._unwrap_exception(content)}) | |
870 | elif msg_type == 'pyin': |
|
849 | elif msg_type == 'pyin': | |
871 | md.update({'pyin' : content['code']}) |
|
850 | md.update({'pyin' : content['code']}) | |
872 | elif msg_type == 'display_data': |
|
851 | elif msg_type == 'display_data': | |
873 | md['outputs'].append(content) |
|
852 | md['outputs'].append(content) | |
874 | elif msg_type == 'pyout': |
|
853 | elif msg_type == 'pyout': | |
875 | md['pyout'] = content |
|
854 | md['pyout'] = content | |
876 | elif msg_type == 'status': |
|
855 | elif msg_type == 'status': | |
877 | # idle message comes after all outputs |
|
856 | # idle message comes after all outputs | |
878 | if content['execution_state'] == 'idle': |
|
857 | if content['execution_state'] == 'idle': | |
879 | md['outputs_ready'] = True |
|
858 | md['outputs_ready'] = True | |
880 | else: |
|
859 | else: | |
881 | # unhandled msg_type (status, etc.) |
|
860 | # unhandled msg_type (status, etc.) | |
882 | pass |
|
861 | pass | |
883 |
|
862 | |||
884 | # reduntant? |
|
863 | # reduntant? | |
885 | self.metadata[msg_id] = md |
|
864 | self.metadata[msg_id] = md | |
886 |
|
865 | |||
887 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) |
|
866 | idents,msg = self.session.recv(sock, mode=zmq.NOBLOCK) | |
888 |
|
867 | |||
889 | #-------------------------------------------------------------------------- |
|
868 | #-------------------------------------------------------------------------- | |
890 | # len, getitem |
|
869 | # len, getitem | |
891 | #-------------------------------------------------------------------------- |
|
870 | #-------------------------------------------------------------------------- | |
892 |
|
871 | |||
893 | def __len__(self): |
|
872 | def __len__(self): | |
894 | """len(client) returns # of engines.""" |
|
873 | """len(client) returns # of engines.""" | |
895 | return len(self.ids) |
|
874 | return len(self.ids) | |
896 |
|
875 | |||
897 | def __getitem__(self, key): |
|
876 | def __getitem__(self, key): | |
898 | """index access returns DirectView multiplexer objects |
|
877 | """index access returns DirectView multiplexer objects | |
899 |
|
878 | |||
900 | Must be int, slice, or list/tuple/xrange of ints""" |
|
879 | Must be int, slice, or list/tuple/xrange of ints""" | |
901 | if not isinstance(key, (int, slice, tuple, list, xrange)): |
|
880 | if not isinstance(key, (int, slice, tuple, list, xrange)): | |
902 | raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key))) |
|
881 | raise TypeError("key by int/slice/iterable of ints only, not %s"%(type(key))) | |
903 | else: |
|
882 | else: | |
904 | return self.direct_view(key) |
|
883 | return self.direct_view(key) | |
905 |
|
884 | |||
906 | #-------------------------------------------------------------------------- |
|
885 | #-------------------------------------------------------------------------- | |
907 | # Begin public methods |
|
886 | # Begin public methods | |
908 | #-------------------------------------------------------------------------- |
|
887 | #-------------------------------------------------------------------------- | |
909 |
|
888 | |||
910 | @property |
|
889 | @property | |
911 | def ids(self): |
|
890 | def ids(self): | |
912 | """Always up-to-date ids property.""" |
|
891 | """Always up-to-date ids property.""" | |
913 | self._flush_notifications() |
|
892 | self._flush_notifications() | |
914 | # always copy: |
|
893 | # always copy: | |
915 | return list(self._ids) |
|
894 | return list(self._ids) | |
916 |
|
895 | |||
917 | def activate(self, targets='all', suffix=''): |
|
896 | def activate(self, targets='all', suffix=''): | |
918 | """Create a DirectView and register it with IPython magics |
|
897 | """Create a DirectView and register it with IPython magics | |
919 |
|
898 | |||
920 | Defines the magics `%px, %autopx, %pxresult, %%px` |
|
899 | Defines the magics `%px, %autopx, %pxresult, %%px` | |
921 |
|
900 | |||
922 | Parameters |
|
901 | Parameters | |
923 | ---------- |
|
902 | ---------- | |
924 |
|
903 | |||
925 | targets: int, list of ints, or 'all' |
|
904 | targets: int, list of ints, or 'all' | |
926 | The engines on which the view's magics will run |
|
905 | The engines on which the view's magics will run | |
927 | suffix: str [default: ''] |
|
906 | suffix: str [default: ''] | |
928 | The suffix, if any, for the magics. This allows you to have |
|
907 | The suffix, if any, for the magics. This allows you to have | |
929 | multiple views associated with parallel magics at the same time. |
|
908 | multiple views associated with parallel magics at the same time. | |
930 |
|
909 | |||
931 | e.g. ``rc.activate(targets=0, suffix='0')`` will give you |
|
910 | e.g. ``rc.activate(targets=0, suffix='0')`` will give you | |
932 | the magics ``%px0``, ``%pxresult0``, etc. for running magics just |
|
911 | the magics ``%px0``, ``%pxresult0``, etc. for running magics just | |
933 | on engine 0. |
|
912 | on engine 0. | |
934 | """ |
|
913 | """ | |
935 | view = self.direct_view(targets) |
|
914 | view = self.direct_view(targets) | |
936 | view.block = True |
|
915 | view.block = True | |
937 | view.activate(suffix) |
|
916 | view.activate(suffix) | |
938 | return view |
|
917 | return view | |
939 |
|
918 | |||
940 | def close(self): |
|
919 | def close(self): | |
941 | if self._closed: |
|
920 | if self._closed: | |
942 | return |
|
921 | return | |
943 | self.stop_spin_thread() |
|
922 | self.stop_spin_thread() | |
944 | snames = filter(lambda n: n.endswith('socket'), dir(self)) |
|
923 | snames = filter(lambda n: n.endswith('socket'), dir(self)) | |
945 | for socket in map(lambda name: getattr(self, name), snames): |
|
924 | for socket in map(lambda name: getattr(self, name), snames): | |
946 | if isinstance(socket, zmq.Socket) and not socket.closed: |
|
925 | if isinstance(socket, zmq.Socket) and not socket.closed: | |
947 | socket.close() |
|
926 | socket.close() | |
948 | self._closed = True |
|
927 | self._closed = True | |
949 |
|
928 | |||
950 | def _spin_every(self, interval=1): |
|
929 | def _spin_every(self, interval=1): | |
951 | """target func for use in spin_thread""" |
|
930 | """target func for use in spin_thread""" | |
952 | while True: |
|
931 | while True: | |
953 | if self._stop_spinning.is_set(): |
|
932 | if self._stop_spinning.is_set(): | |
954 | return |
|
933 | return | |
955 | time.sleep(interval) |
|
934 | time.sleep(interval) | |
956 | self.spin() |
|
935 | self.spin() | |
957 |
|
936 | |||
958 | def spin_thread(self, interval=1): |
|
937 | def spin_thread(self, interval=1): | |
959 | """call Client.spin() in a background thread on some regular interval |
|
938 | """call Client.spin() in a background thread on some regular interval | |
960 |
|
939 | |||
961 | This helps ensure that messages don't pile up too much in the zmq queue |
|
940 | This helps ensure that messages don't pile up too much in the zmq queue | |
962 | while you are working on other things, or just leaving an idle terminal. |
|
941 | while you are working on other things, or just leaving an idle terminal. | |
963 |
|
942 | |||
964 | It also helps limit potential padding of the `received` timestamp |
|
943 | It also helps limit potential padding of the `received` timestamp | |
965 | on AsyncResult objects, used for timings. |
|
944 | on AsyncResult objects, used for timings. | |
966 |
|
945 | |||
967 | Parameters |
|
946 | Parameters | |
968 | ---------- |
|
947 | ---------- | |
969 |
|
948 | |||
970 | interval : float, optional |
|
949 | interval : float, optional | |
971 | The interval on which to spin the client in the background thread |
|
950 | The interval on which to spin the client in the background thread | |
972 | (simply passed to time.sleep). |
|
951 | (simply passed to time.sleep). | |
973 |
|
952 | |||
974 | Notes |
|
953 | Notes | |
975 | ----- |
|
954 | ----- | |
976 |
|
955 | |||
977 | For precision timing, you may want to use this method to put a bound |
|
956 | For precision timing, you may want to use this method to put a bound | |
978 | on the jitter (in seconds) in `received` timestamps used |
|
957 | on the jitter (in seconds) in `received` timestamps used | |
979 | in AsyncResult.wall_time. |
|
958 | in AsyncResult.wall_time. | |
980 |
|
959 | |||
981 | """ |
|
960 | """ | |
982 | if self._spin_thread is not None: |
|
961 | if self._spin_thread is not None: | |
983 | self.stop_spin_thread() |
|
962 | self.stop_spin_thread() | |
984 | self._stop_spinning.clear() |
|
963 | self._stop_spinning.clear() | |
985 | self._spin_thread = Thread(target=self._spin_every, args=(interval,)) |
|
964 | self._spin_thread = Thread(target=self._spin_every, args=(interval,)) | |
986 | self._spin_thread.daemon = True |
|
965 | self._spin_thread.daemon = True | |
987 | self._spin_thread.start() |
|
966 | self._spin_thread.start() | |
988 |
|
967 | |||
989 | def stop_spin_thread(self): |
|
968 | def stop_spin_thread(self): | |
990 | """stop background spin_thread, if any""" |
|
969 | """stop background spin_thread, if any""" | |
991 | if self._spin_thread is not None: |
|
970 | if self._spin_thread is not None: | |
992 | self._stop_spinning.set() |
|
971 | self._stop_spinning.set() | |
993 | self._spin_thread.join() |
|
972 | self._spin_thread.join() | |
994 | self._spin_thread = None |
|
973 | self._spin_thread = None | |
995 |
|
974 | |||
996 | def spin(self): |
|
975 | def spin(self): | |
997 | """Flush any registration notifications and execution results |
|
976 | """Flush any registration notifications and execution results | |
998 | waiting in the ZMQ queue. |
|
977 | waiting in the ZMQ queue. | |
999 | """ |
|
978 | """ | |
1000 | if self._notification_socket: |
|
979 | if self._notification_socket: | |
1001 | self._flush_notifications() |
|
980 | self._flush_notifications() | |
1002 | if self._iopub_socket: |
|
981 | if self._iopub_socket: | |
1003 | self._flush_iopub(self._iopub_socket) |
|
982 | self._flush_iopub(self._iopub_socket) | |
1004 | if self._mux_socket: |
|
983 | if self._mux_socket: | |
1005 | self._flush_results(self._mux_socket) |
|
984 | self._flush_results(self._mux_socket) | |
1006 | if self._task_socket: |
|
985 | if self._task_socket: | |
1007 | self._flush_results(self._task_socket) |
|
986 | self._flush_results(self._task_socket) | |
1008 | if self._control_socket: |
|
987 | if self._control_socket: | |
1009 | self._flush_control(self._control_socket) |
|
988 | self._flush_control(self._control_socket) | |
1010 | if self._query_socket: |
|
989 | if self._query_socket: | |
1011 | self._flush_ignored_hub_replies() |
|
990 | self._flush_ignored_hub_replies() | |
1012 |
|
991 | |||
1013 | def wait(self, jobs=None, timeout=-1): |
|
992 | def wait(self, jobs=None, timeout=-1): | |
1014 | """waits on one or more `jobs`, for up to `timeout` seconds. |
|
993 | """waits on one or more `jobs`, for up to `timeout` seconds. | |
1015 |
|
994 | |||
1016 | Parameters |
|
995 | Parameters | |
1017 | ---------- |
|
996 | ---------- | |
1018 |
|
997 | |||
1019 | jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects |
|
998 | jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects | |
1020 | ints are indices to self.history |
|
999 | ints are indices to self.history | |
1021 | strs are msg_ids |
|
1000 | strs are msg_ids | |
1022 | default: wait on all outstanding messages |
|
1001 | default: wait on all outstanding messages | |
1023 | timeout : float |
|
1002 | timeout : float | |
1024 | a time in seconds, after which to give up. |
|
1003 | a time in seconds, after which to give up. | |
1025 | default is -1, which means no timeout |
|
1004 | default is -1, which means no timeout | |
1026 |
|
1005 | |||
1027 | Returns |
|
1006 | Returns | |
1028 | ------- |
|
1007 | ------- | |
1029 |
|
1008 | |||
1030 | True : when all msg_ids are done |
|
1009 | True : when all msg_ids are done | |
1031 | False : timeout reached, some msg_ids still outstanding |
|
1010 | False : timeout reached, some msg_ids still outstanding | |
1032 | """ |
|
1011 | """ | |
1033 | tic = time.time() |
|
1012 | tic = time.time() | |
1034 | if jobs is None: |
|
1013 | if jobs is None: | |
1035 | theids = self.outstanding |
|
1014 | theids = self.outstanding | |
1036 | else: |
|
1015 | else: | |
1037 | if isinstance(jobs, (int, basestring, AsyncResult)): |
|
1016 | if isinstance(jobs, (int, basestring, AsyncResult)): | |
1038 | jobs = [jobs] |
|
1017 | jobs = [jobs] | |
1039 | theids = set() |
|
1018 | theids = set() | |
1040 | for job in jobs: |
|
1019 | for job in jobs: | |
1041 | if isinstance(job, int): |
|
1020 | if isinstance(job, int): | |
1042 | # index access |
|
1021 | # index access | |
1043 | job = self.history[job] |
|
1022 | job = self.history[job] | |
1044 | elif isinstance(job, AsyncResult): |
|
1023 | elif isinstance(job, AsyncResult): | |
1045 | map(theids.add, job.msg_ids) |
|
1024 | map(theids.add, job.msg_ids) | |
1046 | continue |
|
1025 | continue | |
1047 | theids.add(job) |
|
1026 | theids.add(job) | |
1048 | if not theids.intersection(self.outstanding): |
|
1027 | if not theids.intersection(self.outstanding): | |
1049 | return True |
|
1028 | return True | |
1050 | self.spin() |
|
1029 | self.spin() | |
1051 | while theids.intersection(self.outstanding): |
|
1030 | while theids.intersection(self.outstanding): | |
1052 | if timeout >= 0 and ( time.time()-tic ) > timeout: |
|
1031 | if timeout >= 0 and ( time.time()-tic ) > timeout: | |
1053 | break |
|
1032 | break | |
1054 | time.sleep(1e-3) |
|
1033 | time.sleep(1e-3) | |
1055 | self.spin() |
|
1034 | self.spin() | |
1056 | return len(theids.intersection(self.outstanding)) == 0 |
|
1035 | return len(theids.intersection(self.outstanding)) == 0 | |
1057 |
|
1036 | |||
1058 | #-------------------------------------------------------------------------- |
|
1037 | #-------------------------------------------------------------------------- | |
1059 | # Control methods |
|
1038 | # Control methods | |
1060 | #-------------------------------------------------------------------------- |
|
1039 | #-------------------------------------------------------------------------- | |
1061 |
|
1040 | |||
1062 | @spin_first |
|
1041 | @spin_first | |
1063 | def clear(self, targets=None, block=None): |
|
1042 | def clear(self, targets=None, block=None): | |
1064 | """Clear the namespace in target(s).""" |
|
1043 | """Clear the namespace in target(s).""" | |
1065 | block = self.block if block is None else block |
|
1044 | block = self.block if block is None else block | |
1066 | targets = self._build_targets(targets)[0] |
|
1045 | targets = self._build_targets(targets)[0] | |
1067 | for t in targets: |
|
1046 | for t in targets: | |
1068 | self.session.send(self._control_socket, 'clear_request', content={}, ident=t) |
|
1047 | self.session.send(self._control_socket, 'clear_request', content={}, ident=t) | |
1069 | error = False |
|
1048 | error = False | |
1070 | if block: |
|
1049 | if block: | |
1071 | self._flush_ignored_control() |
|
1050 | self._flush_ignored_control() | |
1072 | for i in range(len(targets)): |
|
1051 | for i in range(len(targets)): | |
1073 | idents,msg = self.session.recv(self._control_socket,0) |
|
1052 | idents,msg = self.session.recv(self._control_socket,0) | |
1074 | if self.debug: |
|
1053 | if self.debug: | |
1075 | pprint(msg) |
|
1054 | pprint(msg) | |
1076 | if msg['content']['status'] != 'ok': |
|
1055 | if msg['content']['status'] != 'ok': | |
1077 | error = self._unwrap_exception(msg['content']) |
|
1056 | error = self._unwrap_exception(msg['content']) | |
1078 | else: |
|
1057 | else: | |
1079 | self._ignored_control_replies += len(targets) |
|
1058 | self._ignored_control_replies += len(targets) | |
1080 | if error: |
|
1059 | if error: | |
1081 | raise error |
|
1060 | raise error | |
1082 |
|
1061 | |||
1083 |
|
1062 | |||
1084 | @spin_first |
|
1063 | @spin_first | |
1085 | def abort(self, jobs=None, targets=None, block=None): |
|
1064 | def abort(self, jobs=None, targets=None, block=None): | |
1086 | """Abort specific jobs from the execution queues of target(s). |
|
1065 | """Abort specific jobs from the execution queues of target(s). | |
1087 |
|
1066 | |||
1088 | This is a mechanism to prevent jobs that have already been submitted |
|
1067 | This is a mechanism to prevent jobs that have already been submitted | |
1089 | from executing. |
|
1068 | from executing. | |
1090 |
|
1069 | |||
1091 | Parameters |
|
1070 | Parameters | |
1092 | ---------- |
|
1071 | ---------- | |
1093 |
|
1072 | |||
1094 | jobs : msg_id, list of msg_ids, or AsyncResult |
|
1073 | jobs : msg_id, list of msg_ids, or AsyncResult | |
1095 | The jobs to be aborted |
|
1074 | The jobs to be aborted | |
1096 |
|
1075 | |||
1097 | If unspecified/None: abort all outstanding jobs. |
|
1076 | If unspecified/None: abort all outstanding jobs. | |
1098 |
|
1077 | |||
1099 | """ |
|
1078 | """ | |
1100 | block = self.block if block is None else block |
|
1079 | block = self.block if block is None else block | |
1101 | jobs = jobs if jobs is not None else list(self.outstanding) |
|
1080 | jobs = jobs if jobs is not None else list(self.outstanding) | |
1102 | targets = self._build_targets(targets)[0] |
|
1081 | targets = self._build_targets(targets)[0] | |
1103 |
|
1082 | |||
1104 | msg_ids = [] |
|
1083 | msg_ids = [] | |
1105 | if isinstance(jobs, (basestring,AsyncResult)): |
|
1084 | if isinstance(jobs, (basestring,AsyncResult)): | |
1106 | jobs = [jobs] |
|
1085 | jobs = [jobs] | |
1107 | bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs) |
|
1086 | bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs) | |
1108 | if bad_ids: |
|
1087 | if bad_ids: | |
1109 | raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0]) |
|
1088 | raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0]) | |
1110 | for j in jobs: |
|
1089 | for j in jobs: | |
1111 | if isinstance(j, AsyncResult): |
|
1090 | if isinstance(j, AsyncResult): | |
1112 | msg_ids.extend(j.msg_ids) |
|
1091 | msg_ids.extend(j.msg_ids) | |
1113 | else: |
|
1092 | else: | |
1114 | msg_ids.append(j) |
|
1093 | msg_ids.append(j) | |
1115 | content = dict(msg_ids=msg_ids) |
|
1094 | content = dict(msg_ids=msg_ids) | |
1116 | for t in targets: |
|
1095 | for t in targets: | |
1117 | self.session.send(self._control_socket, 'abort_request', |
|
1096 | self.session.send(self._control_socket, 'abort_request', | |
1118 | content=content, ident=t) |
|
1097 | content=content, ident=t) | |
1119 | error = False |
|
1098 | error = False | |
1120 | if block: |
|
1099 | if block: | |
1121 | self._flush_ignored_control() |
|
1100 | self._flush_ignored_control() | |
1122 | for i in range(len(targets)): |
|
1101 | for i in range(len(targets)): | |
1123 | idents,msg = self.session.recv(self._control_socket,0) |
|
1102 | idents,msg = self.session.recv(self._control_socket,0) | |
1124 | if self.debug: |
|
1103 | if self.debug: | |
1125 | pprint(msg) |
|
1104 | pprint(msg) | |
1126 | if msg['content']['status'] != 'ok': |
|
1105 | if msg['content']['status'] != 'ok': | |
1127 | error = self._unwrap_exception(msg['content']) |
|
1106 | error = self._unwrap_exception(msg['content']) | |
1128 | else: |
|
1107 | else: | |
1129 | self._ignored_control_replies += len(targets) |
|
1108 | self._ignored_control_replies += len(targets) | |
1130 | if error: |
|
1109 | if error: | |
1131 | raise error |
|
1110 | raise error | |
1132 |
|
1111 | |||
1133 | @spin_first |
|
1112 | @spin_first | |
1134 | def shutdown(self, targets='all', restart=False, hub=False, block=None): |
|
1113 | def shutdown(self, targets='all', restart=False, hub=False, block=None): | |
1135 | """Terminates one or more engine processes, optionally including the hub. |
|
1114 | """Terminates one or more engine processes, optionally including the hub. | |
1136 |
|
1115 | |||
1137 | Parameters |
|
1116 | Parameters | |
1138 | ---------- |
|
1117 | ---------- | |
1139 |
|
1118 | |||
1140 | targets: list of ints or 'all' [default: all] |
|
1119 | targets: list of ints or 'all' [default: all] | |
1141 | Which engines to shutdown. |
|
1120 | Which engines to shutdown. | |
1142 | hub: bool [default: False] |
|
1121 | hub: bool [default: False] | |
1143 | Whether to include the Hub. hub=True implies targets='all'. |
|
1122 | Whether to include the Hub. hub=True implies targets='all'. | |
1144 | block: bool [default: self.block] |
|
1123 | block: bool [default: self.block] | |
1145 | Whether to wait for clean shutdown replies or not. |
|
1124 | Whether to wait for clean shutdown replies or not. | |
1146 | restart: bool [default: False] |
|
1125 | restart: bool [default: False] | |
1147 | NOT IMPLEMENTED |
|
1126 | NOT IMPLEMENTED | |
1148 | whether to restart engines after shutting them down. |
|
1127 | whether to restart engines after shutting them down. | |
1149 | """ |
|
1128 | """ | |
1150 |
|
1129 | |||
1151 | if restart: |
|
1130 | if restart: | |
1152 | raise NotImplementedError("Engine restart is not yet implemented") |
|
1131 | raise NotImplementedError("Engine restart is not yet implemented") | |
1153 |
|
1132 | |||
1154 | block = self.block if block is None else block |
|
1133 | block = self.block if block is None else block | |
1155 | if hub: |
|
1134 | if hub: | |
1156 | targets = 'all' |
|
1135 | targets = 'all' | |
1157 | targets = self._build_targets(targets)[0] |
|
1136 | targets = self._build_targets(targets)[0] | |
1158 | for t in targets: |
|
1137 | for t in targets: | |
1159 | self.session.send(self._control_socket, 'shutdown_request', |
|
1138 | self.session.send(self._control_socket, 'shutdown_request', | |
1160 | content={'restart':restart},ident=t) |
|
1139 | content={'restart':restart},ident=t) | |
1161 | error = False |
|
1140 | error = False | |
1162 | if block or hub: |
|
1141 | if block or hub: | |
1163 | self._flush_ignored_control() |
|
1142 | self._flush_ignored_control() | |
1164 | for i in range(len(targets)): |
|
1143 | for i in range(len(targets)): | |
1165 | idents,msg = self.session.recv(self._control_socket, 0) |
|
1144 | idents,msg = self.session.recv(self._control_socket, 0) | |
1166 | if self.debug: |
|
1145 | if self.debug: | |
1167 | pprint(msg) |
|
1146 | pprint(msg) | |
1168 | if msg['content']['status'] != 'ok': |
|
1147 | if msg['content']['status'] != 'ok': | |
1169 | error = self._unwrap_exception(msg['content']) |
|
1148 | error = self._unwrap_exception(msg['content']) | |
1170 | else: |
|
1149 | else: | |
1171 | self._ignored_control_replies += len(targets) |
|
1150 | self._ignored_control_replies += len(targets) | |
1172 |
|
1151 | |||
1173 | if hub: |
|
1152 | if hub: | |
1174 | time.sleep(0.25) |
|
1153 | time.sleep(0.25) | |
1175 | self.session.send(self._query_socket, 'shutdown_request') |
|
1154 | self.session.send(self._query_socket, 'shutdown_request') | |
1176 | idents,msg = self.session.recv(self._query_socket, 0) |
|
1155 | idents,msg = self.session.recv(self._query_socket, 0) | |
1177 | if self.debug: |
|
1156 | if self.debug: | |
1178 | pprint(msg) |
|
1157 | pprint(msg) | |
1179 | if msg['content']['status'] != 'ok': |
|
1158 | if msg['content']['status'] != 'ok': | |
1180 | error = self._unwrap_exception(msg['content']) |
|
1159 | error = self._unwrap_exception(msg['content']) | |
1181 |
|
1160 | |||
1182 | if error: |
|
1161 | if error: | |
1183 | raise error |
|
1162 | raise error | |
1184 |
|
1163 | |||
1185 | #-------------------------------------------------------------------------- |
|
1164 | #-------------------------------------------------------------------------- | |
1186 | # Execution related methods |
|
1165 | # Execution related methods | |
1187 | #-------------------------------------------------------------------------- |
|
1166 | #-------------------------------------------------------------------------- | |
1188 |
|
1167 | |||
1189 | def _maybe_raise(self, result): |
|
1168 | def _maybe_raise(self, result): | |
1190 | """wrapper for maybe raising an exception if apply failed.""" |
|
1169 | """wrapper for maybe raising an exception if apply failed.""" | |
1191 | if isinstance(result, error.RemoteError): |
|
1170 | if isinstance(result, error.RemoteError): | |
1192 | raise result |
|
1171 | raise result | |
1193 |
|
1172 | |||
1194 | return result |
|
1173 | return result | |
1195 |
|
1174 | |||
1196 | def send_apply_request(self, socket, f, args=None, kwargs=None, subheader=None, track=False, |
|
1175 | def send_apply_request(self, socket, f, args=None, kwargs=None, subheader=None, track=False, | |
1197 | ident=None): |
|
1176 | ident=None): | |
1198 | """construct and send an apply message via a socket. |
|
1177 | """construct and send an apply message via a socket. | |
1199 |
|
1178 | |||
1200 | This is the principal method with which all engine execution is performed by views. |
|
1179 | This is the principal method with which all engine execution is performed by views. | |
1201 | """ |
|
1180 | """ | |
1202 |
|
1181 | |||
1203 | if self._closed: |
|
1182 | if self._closed: | |
1204 | raise RuntimeError("Client cannot be used after its sockets have been closed") |
|
1183 | raise RuntimeError("Client cannot be used after its sockets have been closed") | |
1205 |
|
1184 | |||
1206 | # defaults: |
|
1185 | # defaults: | |
1207 | args = args if args is not None else [] |
|
1186 | args = args if args is not None else [] | |
1208 | kwargs = kwargs if kwargs is not None else {} |
|
1187 | kwargs = kwargs if kwargs is not None else {} | |
1209 | subheader = subheader if subheader is not None else {} |
|
1188 | subheader = subheader if subheader is not None else {} | |
1210 |
|
1189 | |||
1211 | # validate arguments |
|
1190 | # validate arguments | |
1212 | if not callable(f) and not isinstance(f, Reference): |
|
1191 | if not callable(f) and not isinstance(f, Reference): | |
1213 | raise TypeError("f must be callable, not %s"%type(f)) |
|
1192 | raise TypeError("f must be callable, not %s"%type(f)) | |
1214 | if not isinstance(args, (tuple, list)): |
|
1193 | if not isinstance(args, (tuple, list)): | |
1215 | raise TypeError("args must be tuple or list, not %s"%type(args)) |
|
1194 | raise TypeError("args must be tuple or list, not %s"%type(args)) | |
1216 | if not isinstance(kwargs, dict): |
|
1195 | if not isinstance(kwargs, dict): | |
1217 | raise TypeError("kwargs must be dict, not %s"%type(kwargs)) |
|
1196 | raise TypeError("kwargs must be dict, not %s"%type(kwargs)) | |
1218 | if not isinstance(subheader, dict): |
|
1197 | if not isinstance(subheader, dict): | |
1219 | raise TypeError("subheader must be dict, not %s"%type(subheader)) |
|
1198 | raise TypeError("subheader must be dict, not %s"%type(subheader)) | |
1220 |
|
1199 | |||
1221 | bufs = util.pack_apply_message(f,args,kwargs) |
|
1200 | bufs = util.pack_apply_message(f,args,kwargs) | |
1222 |
|
1201 | |||
1223 | msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident, |
|
1202 | msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident, | |
1224 | subheader=subheader, track=track) |
|
1203 | subheader=subheader, track=track) | |
1225 |
|
1204 | |||
1226 | msg_id = msg['header']['msg_id'] |
|
1205 | msg_id = msg['header']['msg_id'] | |
1227 | self.outstanding.add(msg_id) |
|
1206 | self.outstanding.add(msg_id) | |
1228 | if ident: |
|
1207 | if ident: | |
1229 | # possibly routed to a specific engine |
|
1208 | # possibly routed to a specific engine | |
1230 | if isinstance(ident, list): |
|
1209 | if isinstance(ident, list): | |
1231 | ident = ident[-1] |
|
1210 | ident = ident[-1] | |
1232 | if ident in self._engines.values(): |
|
1211 | if ident in self._engines.values(): | |
1233 | # save for later, in case of engine death |
|
1212 | # save for later, in case of engine death | |
1234 | self._outstanding_dict[ident].add(msg_id) |
|
1213 | self._outstanding_dict[ident].add(msg_id) | |
1235 | self.history.append(msg_id) |
|
1214 | self.history.append(msg_id) | |
1236 | self.metadata[msg_id]['submitted'] = datetime.now() |
|
1215 | self.metadata[msg_id]['submitted'] = datetime.now() | |
1237 |
|
1216 | |||
1238 | return msg |
|
1217 | return msg | |
1239 |
|
1218 | |||
1240 | def send_execute_request(self, socket, code, silent=True, subheader=None, ident=None): |
|
1219 | def send_execute_request(self, socket, code, silent=True, subheader=None, ident=None): | |
1241 | """construct and send an execute request via a socket. |
|
1220 | """construct and send an execute request via a socket. | |
1242 |
|
1221 | |||
1243 | """ |
|
1222 | """ | |
1244 |
|
1223 | |||
1245 | if self._closed: |
|
1224 | if self._closed: | |
1246 | raise RuntimeError("Client cannot be used after its sockets have been closed") |
|
1225 | raise RuntimeError("Client cannot be used after its sockets have been closed") | |
1247 |
|
1226 | |||
1248 | # defaults: |
|
1227 | # defaults: | |
1249 | subheader = subheader if subheader is not None else {} |
|
1228 | subheader = subheader if subheader is not None else {} | |
1250 |
|
1229 | |||
1251 | # validate arguments |
|
1230 | # validate arguments | |
1252 | if not isinstance(code, basestring): |
|
1231 | if not isinstance(code, basestring): | |
1253 | raise TypeError("code must be text, not %s" % type(code)) |
|
1232 | raise TypeError("code must be text, not %s" % type(code)) | |
1254 | if not isinstance(subheader, dict): |
|
1233 | if not isinstance(subheader, dict): | |
1255 | raise TypeError("subheader must be dict, not %s" % type(subheader)) |
|
1234 | raise TypeError("subheader must be dict, not %s" % type(subheader)) | |
1256 |
|
1235 | |||
1257 | content = dict(code=code, silent=bool(silent), user_variables=[], user_expressions={}) |
|
1236 | content = dict(code=code, silent=bool(silent), user_variables=[], user_expressions={}) | |
1258 |
|
1237 | |||
1259 |
|
1238 | |||
1260 | msg = self.session.send(socket, "execute_request", content=content, ident=ident, |
|
1239 | msg = self.session.send(socket, "execute_request", content=content, ident=ident, | |
1261 | subheader=subheader) |
|
1240 | subheader=subheader) | |
1262 |
|
1241 | |||
1263 | msg_id = msg['header']['msg_id'] |
|
1242 | msg_id = msg['header']['msg_id'] | |
1264 | self.outstanding.add(msg_id) |
|
1243 | self.outstanding.add(msg_id) | |
1265 | if ident: |
|
1244 | if ident: | |
1266 | # possibly routed to a specific engine |
|
1245 | # possibly routed to a specific engine | |
1267 | if isinstance(ident, list): |
|
1246 | if isinstance(ident, list): | |
1268 | ident = ident[-1] |
|
1247 | ident = ident[-1] | |
1269 | if ident in self._engines.values(): |
|
1248 | if ident in self._engines.values(): | |
1270 | # save for later, in case of engine death |
|
1249 | # save for later, in case of engine death | |
1271 | self._outstanding_dict[ident].add(msg_id) |
|
1250 | self._outstanding_dict[ident].add(msg_id) | |
1272 | self.history.append(msg_id) |
|
1251 | self.history.append(msg_id) | |
1273 | self.metadata[msg_id]['submitted'] = datetime.now() |
|
1252 | self.metadata[msg_id]['submitted'] = datetime.now() | |
1274 |
|
1253 | |||
1275 | return msg |
|
1254 | return msg | |
1276 |
|
1255 | |||
1277 | #-------------------------------------------------------------------------- |
|
1256 | #-------------------------------------------------------------------------- | |
1278 | # construct a View object |
|
1257 | # construct a View object | |
1279 | #-------------------------------------------------------------------------- |
|
1258 | #-------------------------------------------------------------------------- | |
1280 |
|
1259 | |||
1281 | def load_balanced_view(self, targets=None): |
|
1260 | def load_balanced_view(self, targets=None): | |
1282 | """construct a DirectView object. |
|
1261 | """construct a DirectView object. | |
1283 |
|
1262 | |||
1284 | If no arguments are specified, create a LoadBalancedView |
|
1263 | If no arguments are specified, create a LoadBalancedView | |
1285 | using all engines. |
|
1264 | using all engines. | |
1286 |
|
1265 | |||
1287 | Parameters |
|
1266 | Parameters | |
1288 | ---------- |
|
1267 | ---------- | |
1289 |
|
1268 | |||
1290 | targets: list,slice,int,etc. [default: use all engines] |
|
1269 | targets: list,slice,int,etc. [default: use all engines] | |
1291 | The subset of engines across which to load-balance |
|
1270 | The subset of engines across which to load-balance | |
1292 | """ |
|
1271 | """ | |
1293 | if targets == 'all': |
|
1272 | if targets == 'all': | |
1294 | targets = None |
|
1273 | targets = None | |
1295 | if targets is not None: |
|
1274 | if targets is not None: | |
1296 | targets = self._build_targets(targets)[1] |
|
1275 | targets = self._build_targets(targets)[1] | |
1297 | return LoadBalancedView(client=self, socket=self._task_socket, targets=targets) |
|
1276 | return LoadBalancedView(client=self, socket=self._task_socket, targets=targets) | |
1298 |
|
1277 | |||
1299 | def direct_view(self, targets='all'): |
|
1278 | def direct_view(self, targets='all'): | |
1300 | """construct a DirectView object. |
|
1279 | """construct a DirectView object. | |
1301 |
|
1280 | |||
1302 | If no targets are specified, create a DirectView using all engines. |
|
1281 | If no targets are specified, create a DirectView using all engines. | |
1303 |
|
1282 | |||
1304 | rc.direct_view('all') is distinguished from rc[:] in that 'all' will |
|
1283 | rc.direct_view('all') is distinguished from rc[:] in that 'all' will | |
1305 | evaluate the target engines at each execution, whereas rc[:] will connect to |
|
1284 | evaluate the target engines at each execution, whereas rc[:] will connect to | |
1306 | all *current* engines, and that list will not change. |
|
1285 | all *current* engines, and that list will not change. | |
1307 |
|
1286 | |||
1308 | That is, 'all' will always use all engines, whereas rc[:] will not use |
|
1287 | That is, 'all' will always use all engines, whereas rc[:] will not use | |
1309 | engines added after the DirectView is constructed. |
|
1288 | engines added after the DirectView is constructed. | |
1310 |
|
1289 | |||
1311 | Parameters |
|
1290 | Parameters | |
1312 | ---------- |
|
1291 | ---------- | |
1313 |
|
1292 | |||
1314 | targets: list,slice,int,etc. [default: use all engines] |
|
1293 | targets: list,slice,int,etc. [default: use all engines] | |
1315 | The engines to use for the View |
|
1294 | The engines to use for the View | |
1316 | """ |
|
1295 | """ | |
1317 | single = isinstance(targets, int) |
|
1296 | single = isinstance(targets, int) | |
1318 | # allow 'all' to be lazily evaluated at each execution |
|
1297 | # allow 'all' to be lazily evaluated at each execution | |
1319 | if targets != 'all': |
|
1298 | if targets != 'all': | |
1320 | targets = self._build_targets(targets)[1] |
|
1299 | targets = self._build_targets(targets)[1] | |
1321 | if single: |
|
1300 | if single: | |
1322 | targets = targets[0] |
|
1301 | targets = targets[0] | |
1323 | return DirectView(client=self, socket=self._mux_socket, targets=targets) |
|
1302 | return DirectView(client=self, socket=self._mux_socket, targets=targets) | |
1324 |
|
1303 | |||
1325 | #-------------------------------------------------------------------------- |
|
1304 | #-------------------------------------------------------------------------- | |
1326 | # Query methods |
|
1305 | # Query methods | |
1327 | #-------------------------------------------------------------------------- |
|
1306 | #-------------------------------------------------------------------------- | |
1328 |
|
1307 | |||
1329 | @spin_first |
|
1308 | @spin_first | |
1330 | def get_result(self, indices_or_msg_ids=None, block=None): |
|
1309 | def get_result(self, indices_or_msg_ids=None, block=None): | |
1331 | """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object. |
|
1310 | """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object. | |
1332 |
|
1311 | |||
1333 | If the client already has the results, no request to the Hub will be made. |
|
1312 | If the client already has the results, no request to the Hub will be made. | |
1334 |
|
1313 | |||
1335 | This is a convenient way to construct AsyncResult objects, which are wrappers |
|
1314 | This is a convenient way to construct AsyncResult objects, which are wrappers | |
1336 | that include metadata about execution, and allow for awaiting results that |
|
1315 | that include metadata about execution, and allow for awaiting results that | |
1337 | were not submitted by this Client. |
|
1316 | were not submitted by this Client. | |
1338 |
|
1317 | |||
1339 | It can also be a convenient way to retrieve the metadata associated with |
|
1318 | It can also be a convenient way to retrieve the metadata associated with | |
1340 | blocking execution, since it always retrieves |
|
1319 | blocking execution, since it always retrieves | |
1341 |
|
1320 | |||
1342 | Examples |
|
1321 | Examples | |
1343 | -------- |
|
1322 | -------- | |
1344 | :: |
|
1323 | :: | |
1345 |
|
1324 | |||
1346 | In [10]: r = client.apply() |
|
1325 | In [10]: r = client.apply() | |
1347 |
|
1326 | |||
1348 | Parameters |
|
1327 | Parameters | |
1349 | ---------- |
|
1328 | ---------- | |
1350 |
|
1329 | |||
1351 | indices_or_msg_ids : integer history index, str msg_id, or list of either |
|
1330 | indices_or_msg_ids : integer history index, str msg_id, or list of either | |
1352 | The indices or msg_ids of indices to be retrieved |
|
1331 | The indices or msg_ids of indices to be retrieved | |
1353 |
|
1332 | |||
1354 | block : bool |
|
1333 | block : bool | |
1355 | Whether to wait for the result to be done |
|
1334 | Whether to wait for the result to be done | |
1356 |
|
1335 | |||
1357 | Returns |
|
1336 | Returns | |
1358 | ------- |
|
1337 | ------- | |
1359 |
|
1338 | |||
1360 | AsyncResult |
|
1339 | AsyncResult | |
1361 | A single AsyncResult object will always be returned. |
|
1340 | A single AsyncResult object will always be returned. | |
1362 |
|
1341 | |||
1363 | AsyncHubResult |
|
1342 | AsyncHubResult | |
1364 | A subclass of AsyncResult that retrieves results from the Hub |
|
1343 | A subclass of AsyncResult that retrieves results from the Hub | |
1365 |
|
1344 | |||
1366 | """ |
|
1345 | """ | |
1367 | block = self.block if block is None else block |
|
1346 | block = self.block if block is None else block | |
1368 | if indices_or_msg_ids is None: |
|
1347 | if indices_or_msg_ids is None: | |
1369 | indices_or_msg_ids = -1 |
|
1348 | indices_or_msg_ids = -1 | |
1370 |
|
1349 | |||
1371 | if not isinstance(indices_or_msg_ids, (list,tuple)): |
|
1350 | if not isinstance(indices_or_msg_ids, (list,tuple)): | |
1372 | indices_or_msg_ids = [indices_or_msg_ids] |
|
1351 | indices_or_msg_ids = [indices_or_msg_ids] | |
1373 |
|
1352 | |||
1374 | theids = [] |
|
1353 | theids = [] | |
1375 | for id in indices_or_msg_ids: |
|
1354 | for id in indices_or_msg_ids: | |
1376 | if isinstance(id, int): |
|
1355 | if isinstance(id, int): | |
1377 | id = self.history[id] |
|
1356 | id = self.history[id] | |
1378 | if not isinstance(id, basestring): |
|
1357 | if not isinstance(id, basestring): | |
1379 | raise TypeError("indices must be str or int, not %r"%id) |
|
1358 | raise TypeError("indices must be str or int, not %r"%id) | |
1380 | theids.append(id) |
|
1359 | theids.append(id) | |
1381 |
|
1360 | |||
1382 | local_ids = filter(lambda msg_id: msg_id in self.history or msg_id in self.results, theids) |
|
1361 | local_ids = filter(lambda msg_id: msg_id in self.history or msg_id in self.results, theids) | |
1383 | remote_ids = filter(lambda msg_id: msg_id not in local_ids, theids) |
|
1362 | remote_ids = filter(lambda msg_id: msg_id not in local_ids, theids) | |
1384 |
|
1363 | |||
1385 | if remote_ids: |
|
1364 | if remote_ids: | |
1386 | ar = AsyncHubResult(self, msg_ids=theids) |
|
1365 | ar = AsyncHubResult(self, msg_ids=theids) | |
1387 | else: |
|
1366 | else: | |
1388 | ar = AsyncResult(self, msg_ids=theids) |
|
1367 | ar = AsyncResult(self, msg_ids=theids) | |
1389 |
|
1368 | |||
1390 | if block: |
|
1369 | if block: | |
1391 | ar.wait() |
|
1370 | ar.wait() | |
1392 |
|
1371 | |||
1393 | return ar |
|
1372 | return ar | |
1394 |
|
1373 | |||
1395 | @spin_first |
|
1374 | @spin_first | |
1396 | def resubmit(self, indices_or_msg_ids=None, subheader=None, block=None): |
|
1375 | def resubmit(self, indices_or_msg_ids=None, subheader=None, block=None): | |
1397 | """Resubmit one or more tasks. |
|
1376 | """Resubmit one or more tasks. | |
1398 |
|
1377 | |||
1399 | in-flight tasks may not be resubmitted. |
|
1378 | in-flight tasks may not be resubmitted. | |
1400 |
|
1379 | |||
1401 | Parameters |
|
1380 | Parameters | |
1402 | ---------- |
|
1381 | ---------- | |
1403 |
|
1382 | |||
1404 | indices_or_msg_ids : integer history index, str msg_id, or list of either |
|
1383 | indices_or_msg_ids : integer history index, str msg_id, or list of either | |
1405 | The indices or msg_ids of indices to be retrieved |
|
1384 | The indices or msg_ids of indices to be retrieved | |
1406 |
|
1385 | |||
1407 | block : bool |
|
1386 | block : bool | |
1408 | Whether to wait for the result to be done |
|
1387 | Whether to wait for the result to be done | |
1409 |
|
1388 | |||
1410 | Returns |
|
1389 | Returns | |
1411 | ------- |
|
1390 | ------- | |
1412 |
|
1391 | |||
1413 | AsyncHubResult |
|
1392 | AsyncHubResult | |
1414 | A subclass of AsyncResult that retrieves results from the Hub |
|
1393 | A subclass of AsyncResult that retrieves results from the Hub | |
1415 |
|
1394 | |||
1416 | """ |
|
1395 | """ | |
1417 | block = self.block if block is None else block |
|
1396 | block = self.block if block is None else block | |
1418 | if indices_or_msg_ids is None: |
|
1397 | if indices_or_msg_ids is None: | |
1419 | indices_or_msg_ids = -1 |
|
1398 | indices_or_msg_ids = -1 | |
1420 |
|
1399 | |||
1421 | if not isinstance(indices_or_msg_ids, (list,tuple)): |
|
1400 | if not isinstance(indices_or_msg_ids, (list,tuple)): | |
1422 | indices_or_msg_ids = [indices_or_msg_ids] |
|
1401 | indices_or_msg_ids = [indices_or_msg_ids] | |
1423 |
|
1402 | |||
1424 | theids = [] |
|
1403 | theids = [] | |
1425 | for id in indices_or_msg_ids: |
|
1404 | for id in indices_or_msg_ids: | |
1426 | if isinstance(id, int): |
|
1405 | if isinstance(id, int): | |
1427 | id = self.history[id] |
|
1406 | id = self.history[id] | |
1428 | if not isinstance(id, basestring): |
|
1407 | if not isinstance(id, basestring): | |
1429 | raise TypeError("indices must be str or int, not %r"%id) |
|
1408 | raise TypeError("indices must be str or int, not %r"%id) | |
1430 | theids.append(id) |
|
1409 | theids.append(id) | |
1431 |
|
1410 | |||
1432 | content = dict(msg_ids = theids) |
|
1411 | content = dict(msg_ids = theids) | |
1433 |
|
1412 | |||
1434 | self.session.send(self._query_socket, 'resubmit_request', content) |
|
1413 | self.session.send(self._query_socket, 'resubmit_request', content) | |
1435 |
|
1414 | |||
1436 | zmq.select([self._query_socket], [], []) |
|
1415 | zmq.select([self._query_socket], [], []) | |
1437 | idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK) |
|
1416 | idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK) | |
1438 | if self.debug: |
|
1417 | if self.debug: | |
1439 | pprint(msg) |
|
1418 | pprint(msg) | |
1440 | content = msg['content'] |
|
1419 | content = msg['content'] | |
1441 | if content['status'] != 'ok': |
|
1420 | if content['status'] != 'ok': | |
1442 | raise self._unwrap_exception(content) |
|
1421 | raise self._unwrap_exception(content) | |
1443 | mapping = content['resubmitted'] |
|
1422 | mapping = content['resubmitted'] | |
1444 | new_ids = [ mapping[msg_id] for msg_id in theids ] |
|
1423 | new_ids = [ mapping[msg_id] for msg_id in theids ] | |
1445 |
|
1424 | |||
1446 | ar = AsyncHubResult(self, msg_ids=new_ids) |
|
1425 | ar = AsyncHubResult(self, msg_ids=new_ids) | |
1447 |
|
1426 | |||
1448 | if block: |
|
1427 | if block: | |
1449 | ar.wait() |
|
1428 | ar.wait() | |
1450 |
|
1429 | |||
1451 | return ar |
|
1430 | return ar | |
1452 |
|
1431 | |||
1453 | @spin_first |
|
1432 | @spin_first | |
1454 | def result_status(self, msg_ids, status_only=True): |
|
1433 | def result_status(self, msg_ids, status_only=True): | |
1455 | """Check on the status of the result(s) of the apply request with `msg_ids`. |
|
1434 | """Check on the status of the result(s) of the apply request with `msg_ids`. | |
1456 |
|
1435 | |||
1457 | If status_only is False, then the actual results will be retrieved, else |
|
1436 | If status_only is False, then the actual results will be retrieved, else | |
1458 | only the status of the results will be checked. |
|
1437 | only the status of the results will be checked. | |
1459 |
|
1438 | |||
1460 | Parameters |
|
1439 | Parameters | |
1461 | ---------- |
|
1440 | ---------- | |
1462 |
|
1441 | |||
1463 | msg_ids : list of msg_ids |
|
1442 | msg_ids : list of msg_ids | |
1464 | if int: |
|
1443 | if int: | |
1465 | Passed as index to self.history for convenience. |
|
1444 | Passed as index to self.history for convenience. | |
1466 | status_only : bool (default: True) |
|
1445 | status_only : bool (default: True) | |
1467 | if False: |
|
1446 | if False: | |
1468 | Retrieve the actual results of completed tasks. |
|
1447 | Retrieve the actual results of completed tasks. | |
1469 |
|
1448 | |||
1470 | Returns |
|
1449 | Returns | |
1471 | ------- |
|
1450 | ------- | |
1472 |
|
1451 | |||
1473 | results : dict |
|
1452 | results : dict | |
1474 | There will always be the keys 'pending' and 'completed', which will |
|
1453 | There will always be the keys 'pending' and 'completed', which will | |
1475 | be lists of msg_ids that are incomplete or complete. If `status_only` |
|
1454 | be lists of msg_ids that are incomplete or complete. If `status_only` | |
1476 | is False, then completed results will be keyed by their `msg_id`. |
|
1455 | is False, then completed results will be keyed by their `msg_id`. | |
1477 | """ |
|
1456 | """ | |
1478 | if not isinstance(msg_ids, (list,tuple)): |
|
1457 | if not isinstance(msg_ids, (list,tuple)): | |
1479 | msg_ids = [msg_ids] |
|
1458 | msg_ids = [msg_ids] | |
1480 |
|
1459 | |||
1481 | theids = [] |
|
1460 | theids = [] | |
1482 | for msg_id in msg_ids: |
|
1461 | for msg_id in msg_ids: | |
1483 | if isinstance(msg_id, int): |
|
1462 | if isinstance(msg_id, int): | |
1484 | msg_id = self.history[msg_id] |
|
1463 | msg_id = self.history[msg_id] | |
1485 | if not isinstance(msg_id, basestring): |
|
1464 | if not isinstance(msg_id, basestring): | |
1486 | raise TypeError("msg_ids must be str, not %r"%msg_id) |
|
1465 | raise TypeError("msg_ids must be str, not %r"%msg_id) | |
1487 | theids.append(msg_id) |
|
1466 | theids.append(msg_id) | |
1488 |
|
1467 | |||
1489 | completed = [] |
|
1468 | completed = [] | |
1490 | local_results = {} |
|
1469 | local_results = {} | |
1491 |
|
1470 | |||
1492 | # comment this block out to temporarily disable local shortcut: |
|
1471 | # comment this block out to temporarily disable local shortcut: | |
1493 | for msg_id in theids: |
|
1472 | for msg_id in theids: | |
1494 | if msg_id in self.results: |
|
1473 | if msg_id in self.results: | |
1495 | completed.append(msg_id) |
|
1474 | completed.append(msg_id) | |
1496 | local_results[msg_id] = self.results[msg_id] |
|
1475 | local_results[msg_id] = self.results[msg_id] | |
1497 | theids.remove(msg_id) |
|
1476 | theids.remove(msg_id) | |
1498 |
|
1477 | |||
1499 | if theids: # some not locally cached |
|
1478 | if theids: # some not locally cached | |
1500 | content = dict(msg_ids=theids, status_only=status_only) |
|
1479 | content = dict(msg_ids=theids, status_only=status_only) | |
1501 | msg = self.session.send(self._query_socket, "result_request", content=content) |
|
1480 | msg = self.session.send(self._query_socket, "result_request", content=content) | |
1502 | zmq.select([self._query_socket], [], []) |
|
1481 | zmq.select([self._query_socket], [], []) | |
1503 | idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK) |
|
1482 | idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK) | |
1504 | if self.debug: |
|
1483 | if self.debug: | |
1505 | pprint(msg) |
|
1484 | pprint(msg) | |
1506 | content = msg['content'] |
|
1485 | content = msg['content'] | |
1507 | if content['status'] != 'ok': |
|
1486 | if content['status'] != 'ok': | |
1508 | raise self._unwrap_exception(content) |
|
1487 | raise self._unwrap_exception(content) | |
1509 | buffers = msg['buffers'] |
|
1488 | buffers = msg['buffers'] | |
1510 | else: |
|
1489 | else: | |
1511 | content = dict(completed=[],pending=[]) |
|
1490 | content = dict(completed=[],pending=[]) | |
1512 |
|
1491 | |||
1513 | content['completed'].extend(completed) |
|
1492 | content['completed'].extend(completed) | |
1514 |
|
1493 | |||
1515 | if status_only: |
|
1494 | if status_only: | |
1516 | return content |
|
1495 | return content | |
1517 |
|
1496 | |||
1518 | failures = [] |
|
1497 | failures = [] | |
1519 | # load cached results into result: |
|
1498 | # load cached results into result: | |
1520 | content.update(local_results) |
|
1499 | content.update(local_results) | |
1521 |
|
1500 | |||
1522 | # update cache with results: |
|
1501 | # update cache with results: | |
1523 | for msg_id in sorted(theids): |
|
1502 | for msg_id in sorted(theids): | |
1524 | if msg_id in content['completed']: |
|
1503 | if msg_id in content['completed']: | |
1525 | rec = content[msg_id] |
|
1504 | rec = content[msg_id] | |
1526 | parent = rec['header'] |
|
1505 | parent = rec['header'] | |
1527 | header = rec['result_header'] |
|
1506 | header = rec['result_header'] | |
1528 | rcontent = rec['result_content'] |
|
1507 | rcontent = rec['result_content'] | |
1529 | iodict = rec['io'] |
|
1508 | iodict = rec['io'] | |
1530 | if isinstance(rcontent, str): |
|
1509 | if isinstance(rcontent, str): | |
1531 | rcontent = self.session.unpack(rcontent) |
|
1510 | rcontent = self.session.unpack(rcontent) | |
1532 |
|
1511 | |||
1533 | md = self.metadata[msg_id] |
|
1512 | md = self.metadata[msg_id] | |
1534 | md.update(self._extract_metadata(header, parent, rcontent)) |
|
1513 | md.update(self._extract_metadata(header, parent, rcontent)) | |
1535 | if rec.get('received'): |
|
1514 | if rec.get('received'): | |
1536 | md['received'] = rec['received'] |
|
1515 | md['received'] = rec['received'] | |
1537 | md.update(iodict) |
|
1516 | md.update(iodict) | |
1538 |
|
1517 | |||
1539 | if rcontent['status'] == 'ok': |
|
1518 | if rcontent['status'] == 'ok': | |
1540 | if header['msg_type'] == 'apply_reply': |
|
1519 | if header['msg_type'] == 'apply_reply': | |
1541 | res,buffers = util.unserialize_object(buffers) |
|
1520 | res,buffers = util.unserialize_object(buffers) | |
1542 | elif header['msg_type'] == 'execute_reply': |
|
1521 | elif header['msg_type'] == 'execute_reply': | |
1543 | res = ExecuteReply(msg_id, rcontent, md) |
|
1522 | res = ExecuteReply(msg_id, rcontent, md) | |
1544 | else: |
|
1523 | else: | |
1545 | raise KeyError("unhandled msg type: %r" % header[msg_type]) |
|
1524 | raise KeyError("unhandled msg type: %r" % header[msg_type]) | |
1546 | else: |
|
1525 | else: | |
1547 | res = self._unwrap_exception(rcontent) |
|
1526 | res = self._unwrap_exception(rcontent) | |
1548 | failures.append(res) |
|
1527 | failures.append(res) | |
1549 |
|
1528 | |||
1550 | self.results[msg_id] = res |
|
1529 | self.results[msg_id] = res | |
1551 | content[msg_id] = res |
|
1530 | content[msg_id] = res | |
1552 |
|
1531 | |||
1553 | if len(theids) == 1 and failures: |
|
1532 | if len(theids) == 1 and failures: | |
1554 | raise failures[0] |
|
1533 | raise failures[0] | |
1555 |
|
1534 | |||
1556 | error.collect_exceptions(failures, "result_status") |
|
1535 | error.collect_exceptions(failures, "result_status") | |
1557 | return content |
|
1536 | return content | |
1558 |
|
1537 | |||
1559 | @spin_first |
|
1538 | @spin_first | |
1560 | def queue_status(self, targets='all', verbose=False): |
|
1539 | def queue_status(self, targets='all', verbose=False): | |
1561 | """Fetch the status of engine queues. |
|
1540 | """Fetch the status of engine queues. | |
1562 |
|
1541 | |||
1563 | Parameters |
|
1542 | Parameters | |
1564 | ---------- |
|
1543 | ---------- | |
1565 |
|
1544 | |||
1566 | targets : int/str/list of ints/strs |
|
1545 | targets : int/str/list of ints/strs | |
1567 | the engines whose states are to be queried. |
|
1546 | the engines whose states are to be queried. | |
1568 | default : all |
|
1547 | default : all | |
1569 | verbose : bool |
|
1548 | verbose : bool | |
1570 | Whether to return lengths only, or lists of ids for each element |
|
1549 | Whether to return lengths only, or lists of ids for each element | |
1571 | """ |
|
1550 | """ | |
1572 | if targets == 'all': |
|
1551 | if targets == 'all': | |
1573 | # allow 'all' to be evaluated on the engine |
|
1552 | # allow 'all' to be evaluated on the engine | |
1574 | engine_ids = None |
|
1553 | engine_ids = None | |
1575 | else: |
|
1554 | else: | |
1576 | engine_ids = self._build_targets(targets)[1] |
|
1555 | engine_ids = self._build_targets(targets)[1] | |
1577 | content = dict(targets=engine_ids, verbose=verbose) |
|
1556 | content = dict(targets=engine_ids, verbose=verbose) | |
1578 | self.session.send(self._query_socket, "queue_request", content=content) |
|
1557 | self.session.send(self._query_socket, "queue_request", content=content) | |
1579 | idents,msg = self.session.recv(self._query_socket, 0) |
|
1558 | idents,msg = self.session.recv(self._query_socket, 0) | |
1580 | if self.debug: |
|
1559 | if self.debug: | |
1581 | pprint(msg) |
|
1560 | pprint(msg) | |
1582 | content = msg['content'] |
|
1561 | content = msg['content'] | |
1583 | status = content.pop('status') |
|
1562 | status = content.pop('status') | |
1584 | if status != 'ok': |
|
1563 | if status != 'ok': | |
1585 | raise self._unwrap_exception(content) |
|
1564 | raise self._unwrap_exception(content) | |
1586 | content = rekey(content) |
|
1565 | content = rekey(content) | |
1587 | if isinstance(targets, int): |
|
1566 | if isinstance(targets, int): | |
1588 | return content[targets] |
|
1567 | return content[targets] | |
1589 | else: |
|
1568 | else: | |
1590 | return content |
|
1569 | return content | |
1591 |
|
1570 | |||
1592 | @spin_first |
|
1571 | @spin_first | |
1593 | def purge_results(self, jobs=[], targets=[]): |
|
1572 | def purge_results(self, jobs=[], targets=[]): | |
1594 | """Tell the Hub to forget results. |
|
1573 | """Tell the Hub to forget results. | |
1595 |
|
1574 | |||
1596 | Individual results can be purged by msg_id, or the entire |
|
1575 | Individual results can be purged by msg_id, or the entire | |
1597 | history of specific targets can be purged. |
|
1576 | history of specific targets can be purged. | |
1598 |
|
1577 | |||
1599 | Use `purge_results('all')` to scrub everything from the Hub's db. |
|
1578 | Use `purge_results('all')` to scrub everything from the Hub's db. | |
1600 |
|
1579 | |||
1601 | Parameters |
|
1580 | Parameters | |
1602 | ---------- |
|
1581 | ---------- | |
1603 |
|
1582 | |||
1604 | jobs : str or list of str or AsyncResult objects |
|
1583 | jobs : str or list of str or AsyncResult objects | |
1605 | the msg_ids whose results should be forgotten. |
|
1584 | the msg_ids whose results should be forgotten. | |
1606 | targets : int/str/list of ints/strs |
|
1585 | targets : int/str/list of ints/strs | |
1607 | The targets, by int_id, whose entire history is to be purged. |
|
1586 | The targets, by int_id, whose entire history is to be purged. | |
1608 |
|
1587 | |||
1609 | default : None |
|
1588 | default : None | |
1610 | """ |
|
1589 | """ | |
1611 | if not targets and not jobs: |
|
1590 | if not targets and not jobs: | |
1612 | raise ValueError("Must specify at least one of `targets` and `jobs`") |
|
1591 | raise ValueError("Must specify at least one of `targets` and `jobs`") | |
1613 | if targets: |
|
1592 | if targets: | |
1614 | targets = self._build_targets(targets)[1] |
|
1593 | targets = self._build_targets(targets)[1] | |
1615 |
|
1594 | |||
1616 | # construct msg_ids from jobs |
|
1595 | # construct msg_ids from jobs | |
1617 | if jobs == 'all': |
|
1596 | if jobs == 'all': | |
1618 | msg_ids = jobs |
|
1597 | msg_ids = jobs | |
1619 | else: |
|
1598 | else: | |
1620 | msg_ids = [] |
|
1599 | msg_ids = [] | |
1621 | if isinstance(jobs, (basestring,AsyncResult)): |
|
1600 | if isinstance(jobs, (basestring,AsyncResult)): | |
1622 | jobs = [jobs] |
|
1601 | jobs = [jobs] | |
1623 | bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs) |
|
1602 | bad_ids = filter(lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs) | |
1624 | if bad_ids: |
|
1603 | if bad_ids: | |
1625 | raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0]) |
|
1604 | raise TypeError("Invalid msg_id type %r, expected str or AsyncResult"%bad_ids[0]) | |
1626 | for j in jobs: |
|
1605 | for j in jobs: | |
1627 | if isinstance(j, AsyncResult): |
|
1606 | if isinstance(j, AsyncResult): | |
1628 | msg_ids.extend(j.msg_ids) |
|
1607 | msg_ids.extend(j.msg_ids) | |
1629 | else: |
|
1608 | else: | |
1630 | msg_ids.append(j) |
|
1609 | msg_ids.append(j) | |
1631 |
|
1610 | |||
1632 | content = dict(engine_ids=targets, msg_ids=msg_ids) |
|
1611 | content = dict(engine_ids=targets, msg_ids=msg_ids) | |
1633 | self.session.send(self._query_socket, "purge_request", content=content) |
|
1612 | self.session.send(self._query_socket, "purge_request", content=content) | |
1634 | idents, msg = self.session.recv(self._query_socket, 0) |
|
1613 | idents, msg = self.session.recv(self._query_socket, 0) | |
1635 | if self.debug: |
|
1614 | if self.debug: | |
1636 | pprint(msg) |
|
1615 | pprint(msg) | |
1637 | content = msg['content'] |
|
1616 | content = msg['content'] | |
1638 | if content['status'] != 'ok': |
|
1617 | if content['status'] != 'ok': | |
1639 | raise self._unwrap_exception(content) |
|
1618 | raise self._unwrap_exception(content) | |
1640 |
|
1619 | |||
1641 | @spin_first |
|
1620 | @spin_first | |
1642 | def hub_history(self): |
|
1621 | def hub_history(self): | |
1643 | """Get the Hub's history |
|
1622 | """Get the Hub's history | |
1644 |
|
1623 | |||
1645 | Just like the Client, the Hub has a history, which is a list of msg_ids. |
|
1624 | Just like the Client, the Hub has a history, which is a list of msg_ids. | |
1646 | This will contain the history of all clients, and, depending on configuration, |
|
1625 | This will contain the history of all clients, and, depending on configuration, | |
1647 | may contain history across multiple cluster sessions. |
|
1626 | may contain history across multiple cluster sessions. | |
1648 |
|
1627 | |||
1649 | Any msg_id returned here is a valid argument to `get_result`. |
|
1628 | Any msg_id returned here is a valid argument to `get_result`. | |
1650 |
|
1629 | |||
1651 | Returns |
|
1630 | Returns | |
1652 | ------- |
|
1631 | ------- | |
1653 |
|
1632 | |||
1654 | msg_ids : list of strs |
|
1633 | msg_ids : list of strs | |
1655 | list of all msg_ids, ordered by task submission time. |
|
1634 | list of all msg_ids, ordered by task submission time. | |
1656 | """ |
|
1635 | """ | |
1657 |
|
1636 | |||
1658 | self.session.send(self._query_socket, "history_request", content={}) |
|
1637 | self.session.send(self._query_socket, "history_request", content={}) | |
1659 | idents, msg = self.session.recv(self._query_socket, 0) |
|
1638 | idents, msg = self.session.recv(self._query_socket, 0) | |
1660 |
|
1639 | |||
1661 | if self.debug: |
|
1640 | if self.debug: | |
1662 | pprint(msg) |
|
1641 | pprint(msg) | |
1663 | content = msg['content'] |
|
1642 | content = msg['content'] | |
1664 | if content['status'] != 'ok': |
|
1643 | if content['status'] != 'ok': | |
1665 | raise self._unwrap_exception(content) |
|
1644 | raise self._unwrap_exception(content) | |
1666 | else: |
|
1645 | else: | |
1667 | return content['history'] |
|
1646 | return content['history'] | |
1668 |
|
1647 | |||
1669 | @spin_first |
|
1648 | @spin_first | |
1670 | def db_query(self, query, keys=None): |
|
1649 | def db_query(self, query, keys=None): | |
1671 | """Query the Hub's TaskRecord database |
|
1650 | """Query the Hub's TaskRecord database | |
1672 |
|
1651 | |||
1673 | This will return a list of task record dicts that match `query` |
|
1652 | This will return a list of task record dicts that match `query` | |
1674 |
|
1653 | |||
1675 | Parameters |
|
1654 | Parameters | |
1676 | ---------- |
|
1655 | ---------- | |
1677 |
|
1656 | |||
1678 | query : mongodb query dict |
|
1657 | query : mongodb query dict | |
1679 | The search dict. See mongodb query docs for details. |
|
1658 | The search dict. See mongodb query docs for details. | |
1680 | keys : list of strs [optional] |
|
1659 | keys : list of strs [optional] | |
1681 | The subset of keys to be returned. The default is to fetch everything but buffers. |
|
1660 | The subset of keys to be returned. The default is to fetch everything but buffers. | |
1682 | 'msg_id' will *always* be included. |
|
1661 | 'msg_id' will *always* be included. | |
1683 | """ |
|
1662 | """ | |
1684 | if isinstance(keys, basestring): |
|
1663 | if isinstance(keys, basestring): | |
1685 | keys = [keys] |
|
1664 | keys = [keys] | |
1686 | content = dict(query=query, keys=keys) |
|
1665 | content = dict(query=query, keys=keys) | |
1687 | self.session.send(self._query_socket, "db_request", content=content) |
|
1666 | self.session.send(self._query_socket, "db_request", content=content) | |
1688 | idents, msg = self.session.recv(self._query_socket, 0) |
|
1667 | idents, msg = self.session.recv(self._query_socket, 0) | |
1689 | if self.debug: |
|
1668 | if self.debug: | |
1690 | pprint(msg) |
|
1669 | pprint(msg) | |
1691 | content = msg['content'] |
|
1670 | content = msg['content'] | |
1692 | if content['status'] != 'ok': |
|
1671 | if content['status'] != 'ok': | |
1693 | raise self._unwrap_exception(content) |
|
1672 | raise self._unwrap_exception(content) | |
1694 |
|
1673 | |||
1695 | records = content['records'] |
|
1674 | records = content['records'] | |
1696 |
|
1675 | |||
1697 | buffer_lens = content['buffer_lens'] |
|
1676 | buffer_lens = content['buffer_lens'] | |
1698 | result_buffer_lens = content['result_buffer_lens'] |
|
1677 | result_buffer_lens = content['result_buffer_lens'] | |
1699 | buffers = msg['buffers'] |
|
1678 | buffers = msg['buffers'] | |
1700 | has_bufs = buffer_lens is not None |
|
1679 | has_bufs = buffer_lens is not None | |
1701 | has_rbufs = result_buffer_lens is not None |
|
1680 | has_rbufs = result_buffer_lens is not None | |
1702 | for i,rec in enumerate(records): |
|
1681 | for i,rec in enumerate(records): | |
1703 | # relink buffers |
|
1682 | # relink buffers | |
1704 | if has_bufs: |
|
1683 | if has_bufs: | |
1705 | blen = buffer_lens[i] |
|
1684 | blen = buffer_lens[i] | |
1706 | rec['buffers'], buffers = buffers[:blen],buffers[blen:] |
|
1685 | rec['buffers'], buffers = buffers[:blen],buffers[blen:] | |
1707 | if has_rbufs: |
|
1686 | if has_rbufs: | |
1708 | blen = result_buffer_lens[i] |
|
1687 | blen = result_buffer_lens[i] | |
1709 | rec['result_buffers'], buffers = buffers[:blen],buffers[blen:] |
|
1688 | rec['result_buffers'], buffers = buffers[:blen],buffers[blen:] | |
1710 |
|
1689 | |||
1711 | return records |
|
1690 | return records | |
1712 |
|
1691 | |||
1713 | __all__ = [ 'Client' ] |
|
1692 | __all__ = [ 'Client' ] |
@@ -1,1338 +1,1342 b'' | |||||
1 | """The IPython Controller Hub with 0MQ |
|
1 | """The IPython Controller Hub with 0MQ | |
2 | This is the master object that handles connections from engines and clients, |
|
2 | This is the master object that handles connections from engines and clients, | |
3 | and monitors traffic through the various queues. |
|
3 | and monitors traffic through the various queues. | |
4 |
|
4 | |||
5 | Authors: |
|
5 | Authors: | |
6 |
|
6 | |||
7 | * Min RK |
|
7 | * Min RK | |
8 | """ |
|
8 | """ | |
9 | #----------------------------------------------------------------------------- |
|
9 | #----------------------------------------------------------------------------- | |
10 | # Copyright (C) 2010-2011 The IPython Development Team |
|
10 | # Copyright (C) 2010-2011 The IPython Development Team | |
11 | # |
|
11 | # | |
12 | # Distributed under the terms of the BSD License. The full license is in |
|
12 | # Distributed under the terms of the BSD License. The full license is in | |
13 | # the file COPYING, distributed as part of this software. |
|
13 | # the file COPYING, distributed as part of this software. | |
14 | #----------------------------------------------------------------------------- |
|
14 | #----------------------------------------------------------------------------- | |
15 |
|
15 | |||
16 | #----------------------------------------------------------------------------- |
|
16 | #----------------------------------------------------------------------------- | |
17 | # Imports |
|
17 | # Imports | |
18 | #----------------------------------------------------------------------------- |
|
18 | #----------------------------------------------------------------------------- | |
19 | from __future__ import print_function |
|
19 | from __future__ import print_function | |
20 |
|
20 | |||
21 | import sys |
|
21 | import sys | |
22 | import time |
|
22 | import time | |
23 | from datetime import datetime |
|
23 | from datetime import datetime | |
24 |
|
24 | |||
25 | import zmq |
|
25 | import zmq | |
26 | from zmq.eventloop import ioloop |
|
26 | from zmq.eventloop import ioloop | |
27 | from zmq.eventloop.zmqstream import ZMQStream |
|
27 | from zmq.eventloop.zmqstream import ZMQStream | |
28 |
|
28 | |||
29 | # internal: |
|
29 | # internal: | |
30 | from IPython.utils.importstring import import_item |
|
30 | from IPython.utils.importstring import import_item | |
31 | from IPython.utils.py3compat import cast_bytes |
|
31 | from IPython.utils.py3compat import cast_bytes | |
32 | from IPython.utils.traitlets import ( |
|
32 | from IPython.utils.traitlets import ( | |
33 | HasTraits, Instance, Integer, Unicode, Dict, Set, Tuple, CBytes, DottedObjectName |
|
33 | HasTraits, Instance, Integer, Unicode, Dict, Set, Tuple, CBytes, DottedObjectName | |
34 | ) |
|
34 | ) | |
35 |
|
35 | |||
36 | from IPython.parallel import error, util |
|
36 | from IPython.parallel import error, util | |
37 | from IPython.parallel.factory import RegistrationFactory |
|
37 | from IPython.parallel.factory import RegistrationFactory | |
38 |
|
38 | |||
39 | from IPython.zmq.session import SessionFactory |
|
39 | from IPython.zmq.session import SessionFactory | |
40 |
|
40 | |||
41 | from .heartmonitor import HeartMonitor |
|
41 | from .heartmonitor import HeartMonitor | |
42 |
|
42 | |||
43 | #----------------------------------------------------------------------------- |
|
43 | #----------------------------------------------------------------------------- | |
44 | # Code |
|
44 | # Code | |
45 | #----------------------------------------------------------------------------- |
|
45 | #----------------------------------------------------------------------------- | |
46 |
|
46 | |||
47 | def _passer(*args, **kwargs): |
|
47 | def _passer(*args, **kwargs): | |
48 | return |
|
48 | return | |
49 |
|
49 | |||
50 | def _printer(*args, **kwargs): |
|
50 | def _printer(*args, **kwargs): | |
51 | print (args) |
|
51 | print (args) | |
52 | print (kwargs) |
|
52 | print (kwargs) | |
53 |
|
53 | |||
54 | def empty_record(): |
|
54 | def empty_record(): | |
55 | """Return an empty dict with all record keys.""" |
|
55 | """Return an empty dict with all record keys.""" | |
56 | return { |
|
56 | return { | |
57 | 'msg_id' : None, |
|
57 | 'msg_id' : None, | |
58 | 'header' : None, |
|
58 | 'header' : None, | |
59 | 'content': None, |
|
59 | 'content': None, | |
60 | 'buffers': None, |
|
60 | 'buffers': None, | |
61 | 'submitted': None, |
|
61 | 'submitted': None, | |
62 | 'client_uuid' : None, |
|
62 | 'client_uuid' : None, | |
63 | 'engine_uuid' : None, |
|
63 | 'engine_uuid' : None, | |
64 | 'started': None, |
|
64 | 'started': None, | |
65 | 'completed': None, |
|
65 | 'completed': None, | |
66 | 'resubmitted': None, |
|
66 | 'resubmitted': None, | |
67 | 'received': None, |
|
67 | 'received': None, | |
68 | 'result_header' : None, |
|
68 | 'result_header' : None, | |
69 | 'result_content' : None, |
|
69 | 'result_content' : None, | |
70 | 'result_buffers' : None, |
|
70 | 'result_buffers' : None, | |
71 | 'queue' : None, |
|
71 | 'queue' : None, | |
72 | 'pyin' : None, |
|
72 | 'pyin' : None, | |
73 | 'pyout': None, |
|
73 | 'pyout': None, | |
74 | 'pyerr': None, |
|
74 | 'pyerr': None, | |
75 | 'stdout': '', |
|
75 | 'stdout': '', | |
76 | 'stderr': '', |
|
76 | 'stderr': '', | |
77 | } |
|
77 | } | |
78 |
|
78 | |||
79 | def init_record(msg): |
|
79 | def init_record(msg): | |
80 | """Initialize a TaskRecord based on a request.""" |
|
80 | """Initialize a TaskRecord based on a request.""" | |
81 | header = msg['header'] |
|
81 | header = msg['header'] | |
82 | return { |
|
82 | return { | |
83 | 'msg_id' : header['msg_id'], |
|
83 | 'msg_id' : header['msg_id'], | |
84 | 'header' : header, |
|
84 | 'header' : header, | |
85 | 'content': msg['content'], |
|
85 | 'content': msg['content'], | |
86 | 'buffers': msg['buffers'], |
|
86 | 'buffers': msg['buffers'], | |
87 | 'submitted': header['date'], |
|
87 | 'submitted': header['date'], | |
88 | 'client_uuid' : None, |
|
88 | 'client_uuid' : None, | |
89 | 'engine_uuid' : None, |
|
89 | 'engine_uuid' : None, | |
90 | 'started': None, |
|
90 | 'started': None, | |
91 | 'completed': None, |
|
91 | 'completed': None, | |
92 | 'resubmitted': None, |
|
92 | 'resubmitted': None, | |
93 | 'received': None, |
|
93 | 'received': None, | |
94 | 'result_header' : None, |
|
94 | 'result_header' : None, | |
95 | 'result_content' : None, |
|
95 | 'result_content' : None, | |
96 | 'result_buffers' : None, |
|
96 | 'result_buffers' : None, | |
97 | 'queue' : None, |
|
97 | 'queue' : None, | |
98 | 'pyin' : None, |
|
98 | 'pyin' : None, | |
99 | 'pyout': None, |
|
99 | 'pyout': None, | |
100 | 'pyerr': None, |
|
100 | 'pyerr': None, | |
101 | 'stdout': '', |
|
101 | 'stdout': '', | |
102 | 'stderr': '', |
|
102 | 'stderr': '', | |
103 | } |
|
103 | } | |
104 |
|
104 | |||
105 |
|
105 | |||
106 | class EngineConnector(HasTraits): |
|
106 | class EngineConnector(HasTraits): | |
107 | """A simple object for accessing the various zmq connections of an object. |
|
107 | """A simple object for accessing the various zmq connections of an object. | |
108 | Attributes are: |
|
108 | Attributes are: | |
109 | id (int): engine ID |
|
109 | id (int): engine ID | |
110 | uuid (str): uuid (unused?) |
|
110 | uuid (str): uuid (unused?) | |
111 | queue (str): identity of queue's DEALER socket |
|
111 | queue (str): identity of queue's DEALER socket | |
112 | registration (str): identity of registration DEALER socket |
|
112 | registration (str): identity of registration DEALER socket | |
113 | heartbeat (str): identity of heartbeat DEALER socket |
|
113 | heartbeat (str): identity of heartbeat DEALER socket | |
114 | """ |
|
114 | """ | |
115 | id=Integer(0) |
|
115 | id=Integer(0) | |
116 | queue=CBytes() |
|
116 | queue=CBytes() | |
117 | control=CBytes() |
|
117 | control=CBytes() | |
118 | registration=CBytes() |
|
118 | registration=CBytes() | |
119 | heartbeat=CBytes() |
|
119 | heartbeat=CBytes() | |
120 | pending=Set() |
|
120 | pending=Set() | |
121 |
|
121 | |||
122 | _db_shortcuts = { |
|
122 | _db_shortcuts = { | |
123 | 'sqlitedb' : 'IPython.parallel.controller.sqlitedb.SQLiteDB', |
|
123 | 'sqlitedb' : 'IPython.parallel.controller.sqlitedb.SQLiteDB', | |
124 | 'mongodb' : 'IPython.parallel.controller.mongodb.MongoDB', |
|
124 | 'mongodb' : 'IPython.parallel.controller.mongodb.MongoDB', | |
125 | 'dictdb' : 'IPython.parallel.controller.dictdb.DictDB', |
|
125 | 'dictdb' : 'IPython.parallel.controller.dictdb.DictDB', | |
126 | 'nodb' : 'IPython.parallel.controller.dictdb.NoDB', |
|
126 | 'nodb' : 'IPython.parallel.controller.dictdb.NoDB', | |
127 | } |
|
127 | } | |
128 |
|
128 | |||
129 | class HubFactory(RegistrationFactory): |
|
129 | class HubFactory(RegistrationFactory): | |
130 | """The Configurable for setting up a Hub.""" |
|
130 | """The Configurable for setting up a Hub.""" | |
131 |
|
131 | |||
132 | # port-pairs for monitoredqueues: |
|
132 | # port-pairs for monitoredqueues: | |
133 | hb = Tuple(Integer,Integer,config=True, |
|
133 | hb = Tuple(Integer,Integer,config=True, | |
134 | help="""DEALER/SUB Port pair for Engine heartbeats""") |
|
134 | help="""DEALER/SUB Port pair for Engine heartbeats""") | |
135 | def _hb_default(self): |
|
135 | def _hb_default(self): | |
136 | return tuple(util.select_random_ports(2)) |
|
136 | return tuple(util.select_random_ports(2)) | |
137 |
|
137 | |||
138 | mux = Tuple(Integer,Integer,config=True, |
|
138 | mux = Tuple(Integer,Integer,config=True, | |
139 | help="""Engine/Client Port pair for MUX queue""") |
|
139 | help="""Engine/Client Port pair for MUX queue""") | |
140 |
|
140 | |||
141 | def _mux_default(self): |
|
141 | def _mux_default(self): | |
142 | return tuple(util.select_random_ports(2)) |
|
142 | return tuple(util.select_random_ports(2)) | |
143 |
|
143 | |||
144 | task = Tuple(Integer,Integer,config=True, |
|
144 | task = Tuple(Integer,Integer,config=True, | |
145 | help="""Engine/Client Port pair for Task queue""") |
|
145 | help="""Engine/Client Port pair for Task queue""") | |
146 | def _task_default(self): |
|
146 | def _task_default(self): | |
147 | return tuple(util.select_random_ports(2)) |
|
147 | return tuple(util.select_random_ports(2)) | |
148 |
|
148 | |||
149 | control = Tuple(Integer,Integer,config=True, |
|
149 | control = Tuple(Integer,Integer,config=True, | |
150 | help="""Engine/Client Port pair for Control queue""") |
|
150 | help="""Engine/Client Port pair for Control queue""") | |
151 |
|
151 | |||
152 | def _control_default(self): |
|
152 | def _control_default(self): | |
153 | return tuple(util.select_random_ports(2)) |
|
153 | return tuple(util.select_random_ports(2)) | |
154 |
|
154 | |||
155 | iopub = Tuple(Integer,Integer,config=True, |
|
155 | iopub = Tuple(Integer,Integer,config=True, | |
156 | help="""Engine/Client Port pair for IOPub relay""") |
|
156 | help="""Engine/Client Port pair for IOPub relay""") | |
157 |
|
157 | |||
158 | def _iopub_default(self): |
|
158 | def _iopub_default(self): | |
159 | return tuple(util.select_random_ports(2)) |
|
159 | return tuple(util.select_random_ports(2)) | |
160 |
|
160 | |||
161 | # single ports: |
|
161 | # single ports: | |
162 | mon_port = Integer(config=True, |
|
162 | mon_port = Integer(config=True, | |
163 | help="""Monitor (SUB) port for queue traffic""") |
|
163 | help="""Monitor (SUB) port for queue traffic""") | |
164 |
|
164 | |||
165 | def _mon_port_default(self): |
|
165 | def _mon_port_default(self): | |
166 | return util.select_random_ports(1)[0] |
|
166 | return util.select_random_ports(1)[0] | |
167 |
|
167 | |||
168 | notifier_port = Integer(config=True, |
|
168 | notifier_port = Integer(config=True, | |
169 | help="""PUB port for sending engine status notifications""") |
|
169 | help="""PUB port for sending engine status notifications""") | |
170 |
|
170 | |||
171 | def _notifier_port_default(self): |
|
171 | def _notifier_port_default(self): | |
172 | return util.select_random_ports(1)[0] |
|
172 | return util.select_random_ports(1)[0] | |
173 |
|
173 | |||
174 | engine_ip = Unicode('127.0.0.1', config=True, |
|
174 | engine_ip = Unicode('127.0.0.1', config=True, | |
175 | help="IP on which to listen for engine connections. [default: loopback]") |
|
175 | help="IP on which to listen for engine connections. [default: loopback]") | |
176 | engine_transport = Unicode('tcp', config=True, |
|
176 | engine_transport = Unicode('tcp', config=True, | |
177 | help="0MQ transport for engine connections. [default: tcp]") |
|
177 | help="0MQ transport for engine connections. [default: tcp]") | |
178 |
|
178 | |||
179 | client_ip = Unicode('127.0.0.1', config=True, |
|
179 | client_ip = Unicode('127.0.0.1', config=True, | |
180 | help="IP on which to listen for client connections. [default: loopback]") |
|
180 | help="IP on which to listen for client connections. [default: loopback]") | |
181 | client_transport = Unicode('tcp', config=True, |
|
181 | client_transport = Unicode('tcp', config=True, | |
182 | help="0MQ transport for client connections. [default : tcp]") |
|
182 | help="0MQ transport for client connections. [default : tcp]") | |
183 |
|
183 | |||
184 | monitor_ip = Unicode('127.0.0.1', config=True, |
|
184 | monitor_ip = Unicode('127.0.0.1', config=True, | |
185 | help="IP on which to listen for monitor messages. [default: loopback]") |
|
185 | help="IP on which to listen for monitor messages. [default: loopback]") | |
186 | monitor_transport = Unicode('tcp', config=True, |
|
186 | monitor_transport = Unicode('tcp', config=True, | |
187 | help="0MQ transport for monitor messages. [default : tcp]") |
|
187 | help="0MQ transport for monitor messages. [default : tcp]") | |
188 |
|
188 | |||
189 | monitor_url = Unicode('') |
|
189 | monitor_url = Unicode('') | |
190 |
|
190 | |||
191 | db_class = DottedObjectName('NoDB', |
|
191 | db_class = DottedObjectName('NoDB', | |
192 | config=True, help="""The class to use for the DB backend |
|
192 | config=True, help="""The class to use for the DB backend | |
193 |
|
193 | |||
194 | Options include: |
|
194 | Options include: | |
195 |
|
195 | |||
196 | SQLiteDB: SQLite |
|
196 | SQLiteDB: SQLite | |
197 | MongoDB : use MongoDB |
|
197 | MongoDB : use MongoDB | |
198 | DictDB : in-memory storage (fastest, but be mindful of memory growth of the Hub) |
|
198 | DictDB : in-memory storage (fastest, but be mindful of memory growth of the Hub) | |
199 | NoDB : disable database altogether (default) |
|
199 | NoDB : disable database altogether (default) | |
200 |
|
200 | |||
201 | """) |
|
201 | """) | |
202 |
|
202 | |||
203 | # not configurable |
|
203 | # not configurable | |
204 | db = Instance('IPython.parallel.controller.dictdb.BaseDB') |
|
204 | db = Instance('IPython.parallel.controller.dictdb.BaseDB') | |
205 | heartmonitor = Instance('IPython.parallel.controller.heartmonitor.HeartMonitor') |
|
205 | heartmonitor = Instance('IPython.parallel.controller.heartmonitor.HeartMonitor') | |
206 |
|
206 | |||
207 | def _ip_changed(self, name, old, new): |
|
207 | def _ip_changed(self, name, old, new): | |
208 | self.engine_ip = new |
|
208 | self.engine_ip = new | |
209 | self.client_ip = new |
|
209 | self.client_ip = new | |
210 | self.monitor_ip = new |
|
210 | self.monitor_ip = new | |
211 | self._update_monitor_url() |
|
211 | self._update_monitor_url() | |
212 |
|
212 | |||
213 | def _update_monitor_url(self): |
|
213 | def _update_monitor_url(self): | |
214 | self.monitor_url = "%s://%s:%i" % (self.monitor_transport, self.monitor_ip, self.mon_port) |
|
214 | self.monitor_url = "%s://%s:%i" % (self.monitor_transport, self.monitor_ip, self.mon_port) | |
215 |
|
215 | |||
216 | def _transport_changed(self, name, old, new): |
|
216 | def _transport_changed(self, name, old, new): | |
217 | self.engine_transport = new |
|
217 | self.engine_transport = new | |
218 | self.client_transport = new |
|
218 | self.client_transport = new | |
219 | self.monitor_transport = new |
|
219 | self.monitor_transport = new | |
220 | self._update_monitor_url() |
|
220 | self._update_monitor_url() | |
221 |
|
221 | |||
222 | def __init__(self, **kwargs): |
|
222 | def __init__(self, **kwargs): | |
223 | super(HubFactory, self).__init__(**kwargs) |
|
223 | super(HubFactory, self).__init__(**kwargs) | |
224 | self._update_monitor_url() |
|
224 | self._update_monitor_url() | |
225 |
|
225 | |||
226 |
|
226 | |||
227 | def construct(self): |
|
227 | def construct(self): | |
228 | self.init_hub() |
|
228 | self.init_hub() | |
229 |
|
229 | |||
230 | def start(self): |
|
230 | def start(self): | |
231 | self.heartmonitor.start() |
|
231 | self.heartmonitor.start() | |
232 | self.log.info("Heartmonitor started") |
|
232 | self.log.info("Heartmonitor started") | |
233 |
|
233 | |||
234 | def init_hub(self): |
|
234 | def init_hub(self): | |
235 | """construct""" |
|
235 | """construct""" | |
236 | client_iface = "%s://%s:" % (self.client_transport, self.client_ip) + "%i" |
|
236 | client_iface = "%s://%s:" % (self.client_transport, self.client_ip) + "%i" | |
237 | engine_iface = "%s://%s:" % (self.engine_transport, self.engine_ip) + "%i" |
|
237 | engine_iface = "%s://%s:" % (self.engine_transport, self.engine_ip) + "%i" | |
238 |
|
238 | |||
239 | ctx = self.context |
|
239 | ctx = self.context | |
240 | loop = self.loop |
|
240 | loop = self.loop | |
241 |
|
241 | |||
|
242 | try: | |||
|
243 | scheme = self.config.TaskScheduler.scheme_name | |||
|
244 | except AttributeError: | |||
|
245 | from .scheduler import TaskScheduler | |||
|
246 | scheme = TaskScheduler.scheme_name.get_default_value() | |||
|
247 | ||||
|
248 | # build connection dicts | |||
|
249 | engine = self.engine_info = { | |||
|
250 | 'registration' : engine_iface % self.regport, | |||
|
251 | 'control' : engine_iface % self.control[1], | |||
|
252 | 'mux' : engine_iface % self.mux[1], | |||
|
253 | 'hb_ping' : engine_iface % self.hb[0], | |||
|
254 | 'hb_pong' : engine_iface % self.hb[1], | |||
|
255 | 'task' : engine_iface % self.task[1], | |||
|
256 | 'iopub' : engine_iface % self.iopub[1], | |||
|
257 | } | |||
|
258 | ||||
|
259 | client = self.client_info = { | |||
|
260 | 'registration' : client_iface % self.regport, | |||
|
261 | 'control' : client_iface % self.control[0], | |||
|
262 | 'mux' : client_iface % self.mux[0], | |||
|
263 | 'task' : client_iface % self.task[0], | |||
|
264 | 'task_scheme' : scheme, | |||
|
265 | 'iopub' : client_iface % self.iopub[0], | |||
|
266 | 'notification' : client_iface % self.notifier_port, | |||
|
267 | } | |||
|
268 | ||||
|
269 | self.log.debug("Hub engine addrs: %s", self.engine_info) | |||
|
270 | self.log.debug("Hub client addrs: %s", self.client_info) | |||
|
271 | ||||
242 | # Registrar socket |
|
272 | # Registrar socket | |
243 | q = ZMQStream(ctx.socket(zmq.ROUTER), loop) |
|
273 | q = ZMQStream(ctx.socket(zmq.ROUTER), loop) | |
244 |
q.bind(client |
|
274 | q.bind(client['registration']) | |
245 | self.log.info("Hub listening on %s for registration.", client_iface % self.regport) |
|
275 | self.log.info("Hub listening on %s for registration.", client_iface % self.regport) | |
246 | if self.client_ip != self.engine_ip: |
|
276 | if self.client_ip != self.engine_ip: | |
247 |
q.bind(engine |
|
277 | q.bind(engine['registration']) | |
248 | self.log.info("Hub listening on %s for registration.", engine_iface % self.regport) |
|
278 | self.log.info("Hub listening on %s for registration.", engine_iface % self.regport) | |
249 |
|
279 | |||
250 | ### Engine connections ### |
|
280 | ### Engine connections ### | |
251 |
|
281 | |||
252 | # heartbeat |
|
282 | # heartbeat | |
253 | hpub = ctx.socket(zmq.PUB) |
|
283 | hpub = ctx.socket(zmq.PUB) | |
254 |
hpub.bind(engine |
|
284 | hpub.bind(engine['hb_ping']) | |
255 | hrep = ctx.socket(zmq.ROUTER) |
|
285 | hrep = ctx.socket(zmq.ROUTER) | |
256 |
hrep.bind(engine |
|
286 | hrep.bind(engine['hb_pong']) | |
257 | self.heartmonitor = HeartMonitor(loop=loop, config=self.config, log=self.log, |
|
287 | self.heartmonitor = HeartMonitor(loop=loop, config=self.config, log=self.log, | |
258 | pingstream=ZMQStream(hpub,loop), |
|
288 | pingstream=ZMQStream(hpub,loop), | |
259 | pongstream=ZMQStream(hrep,loop) |
|
289 | pongstream=ZMQStream(hrep,loop) | |
260 | ) |
|
290 | ) | |
261 |
|
291 | |||
262 | ### Client connections ### |
|
292 | ### Client connections ### | |
|
293 | ||||
263 | # Notifier socket |
|
294 | # Notifier socket | |
264 | n = ZMQStream(ctx.socket(zmq.PUB), loop) |
|
295 | n = ZMQStream(ctx.socket(zmq.PUB), loop) | |
265 |
n.bind(client |
|
296 | n.bind(client['notification']) | |
266 |
|
297 | |||
267 | ### build and launch the queues ### |
|
298 | ### build and launch the queues ### | |
268 |
|
299 | |||
269 | # monitor socket |
|
300 | # monitor socket | |
270 | sub = ctx.socket(zmq.SUB) |
|
301 | sub = ctx.socket(zmq.SUB) | |
271 | sub.setsockopt(zmq.SUBSCRIBE, b"") |
|
302 | sub.setsockopt(zmq.SUBSCRIBE, b"") | |
272 | sub.bind(self.monitor_url) |
|
303 | sub.bind(self.monitor_url) | |
273 | sub.bind('inproc://monitor') |
|
304 | sub.bind('inproc://monitor') | |
274 | sub = ZMQStream(sub, loop) |
|
305 | sub = ZMQStream(sub, loop) | |
275 |
|
306 | |||
276 | # connect the db |
|
307 | # connect the db | |
277 | db_class = _db_shortcuts.get(self.db_class.lower(), self.db_class) |
|
308 | db_class = _db_shortcuts.get(self.db_class.lower(), self.db_class) | |
278 | self.log.info('Hub using DB backend: %r', (db_class.split('.')[-1])) |
|
309 | self.log.info('Hub using DB backend: %r', (db_class.split('.')[-1])) | |
279 | self.db = import_item(str(db_class))(session=self.session.session, |
|
310 | self.db = import_item(str(db_class))(session=self.session.session, | |
280 | config=self.config, log=self.log) |
|
311 | config=self.config, log=self.log) | |
281 | time.sleep(.25) |
|
312 | time.sleep(.25) | |
282 | try: |
|
|||
283 | scheme = self.config.TaskScheduler.scheme_name |
|
|||
284 | except AttributeError: |
|
|||
285 | from .scheduler import TaskScheduler |
|
|||
286 | scheme = TaskScheduler.scheme_name.get_default_value() |
|
|||
287 | # build connection dicts |
|
|||
288 | self.engine_info = { |
|
|||
289 | 'control' : engine_iface%self.control[1], |
|
|||
290 | 'mux': engine_iface%self.mux[1], |
|
|||
291 | 'heartbeat': (engine_iface%self.hb[0], engine_iface%self.hb[1]), |
|
|||
292 | 'task' : engine_iface%self.task[1], |
|
|||
293 | 'iopub' : engine_iface%self.iopub[1], |
|
|||
294 | # 'monitor' : engine_iface%self.mon_port, |
|
|||
295 | } |
|
|||
296 |
|
||||
297 | self.client_info = { |
|
|||
298 | 'control' : client_iface%self.control[0], |
|
|||
299 | 'mux': client_iface%self.mux[0], |
|
|||
300 | 'task' : (scheme, client_iface%self.task[0]), |
|
|||
301 | 'iopub' : client_iface%self.iopub[0], |
|
|||
302 | 'notification': client_iface%self.notifier_port |
|
|||
303 | } |
|
|||
304 | self.log.debug("Hub engine addrs: %s", self.engine_info) |
|
|||
305 | self.log.debug("Hub client addrs: %s", self.client_info) |
|
|||
306 |
|
313 | |||
307 | # resubmit stream |
|
314 | # resubmit stream | |
308 | r = ZMQStream(ctx.socket(zmq.DEALER), loop) |
|
315 | r = ZMQStream(ctx.socket(zmq.DEALER), loop) | |
309 |
url = util.disambiguate_url(self.client_info['task'] |
|
316 | url = util.disambiguate_url(self.client_info['task']) | |
310 | r.setsockopt(zmq.IDENTITY, self.session.bsession) |
|
|||
311 | r.connect(url) |
|
317 | r.connect(url) | |
312 |
|
318 | |||
313 | self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor, |
|
319 | self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor, | |
314 | query=q, notifier=n, resubmit=r, db=self.db, |
|
320 | query=q, notifier=n, resubmit=r, db=self.db, | |
315 | engine_info=self.engine_info, client_info=self.client_info, |
|
321 | engine_info=self.engine_info, client_info=self.client_info, | |
316 | log=self.log) |
|
322 | log=self.log) | |
317 |
|
323 | |||
318 |
|
324 | |||
319 | class Hub(SessionFactory): |
|
325 | class Hub(SessionFactory): | |
320 | """The IPython Controller Hub with 0MQ connections |
|
326 | """The IPython Controller Hub with 0MQ connections | |
321 |
|
327 | |||
322 | Parameters |
|
328 | Parameters | |
323 | ========== |
|
329 | ========== | |
324 | loop: zmq IOLoop instance |
|
330 | loop: zmq IOLoop instance | |
325 | session: Session object |
|
331 | session: Session object | |
326 | <removed> context: zmq context for creating new connections (?) |
|
332 | <removed> context: zmq context for creating new connections (?) | |
327 | queue: ZMQStream for monitoring the command queue (SUB) |
|
333 | queue: ZMQStream for monitoring the command queue (SUB) | |
328 | query: ZMQStream for engine registration and client queries requests (ROUTER) |
|
334 | query: ZMQStream for engine registration and client queries requests (ROUTER) | |
329 | heartbeat: HeartMonitor object checking the pulse of the engines |
|
335 | heartbeat: HeartMonitor object checking the pulse of the engines | |
330 | notifier: ZMQStream for broadcasting engine registration changes (PUB) |
|
336 | notifier: ZMQStream for broadcasting engine registration changes (PUB) | |
331 | db: connection to db for out of memory logging of commands |
|
337 | db: connection to db for out of memory logging of commands | |
332 | NotImplemented |
|
338 | NotImplemented | |
333 | engine_info: dict of zmq connection information for engines to connect |
|
339 | engine_info: dict of zmq connection information for engines to connect | |
334 | to the queues. |
|
340 | to the queues. | |
335 | client_info: dict of zmq connection information for engines to connect |
|
341 | client_info: dict of zmq connection information for engines to connect | |
336 | to the queues. |
|
342 | to the queues. | |
337 | """ |
|
343 | """ | |
338 | # internal data structures: |
|
344 | # internal data structures: | |
339 | ids=Set() # engine IDs |
|
345 | ids=Set() # engine IDs | |
340 | keytable=Dict() |
|
346 | keytable=Dict() | |
341 | by_ident=Dict() |
|
347 | by_ident=Dict() | |
342 | engines=Dict() |
|
348 | engines=Dict() | |
343 | clients=Dict() |
|
349 | clients=Dict() | |
344 | hearts=Dict() |
|
350 | hearts=Dict() | |
345 | pending=Set() |
|
351 | pending=Set() | |
346 | queues=Dict() # pending msg_ids keyed by engine_id |
|
352 | queues=Dict() # pending msg_ids keyed by engine_id | |
347 | tasks=Dict() # pending msg_ids submitted as tasks, keyed by client_id |
|
353 | tasks=Dict() # pending msg_ids submitted as tasks, keyed by client_id | |
348 | completed=Dict() # completed msg_ids keyed by engine_id |
|
354 | completed=Dict() # completed msg_ids keyed by engine_id | |
349 | all_completed=Set() # completed msg_ids keyed by engine_id |
|
355 | all_completed=Set() # completed msg_ids keyed by engine_id | |
350 | dead_engines=Set() # completed msg_ids keyed by engine_id |
|
356 | dead_engines=Set() # completed msg_ids keyed by engine_id | |
351 | unassigned=Set() # set of task msg_ds not yet assigned a destination |
|
357 | unassigned=Set() # set of task msg_ds not yet assigned a destination | |
352 | incoming_registrations=Dict() |
|
358 | incoming_registrations=Dict() | |
353 | registration_timeout=Integer() |
|
359 | registration_timeout=Integer() | |
354 | _idcounter=Integer(0) |
|
360 | _idcounter=Integer(0) | |
355 |
|
361 | |||
356 | # objects from constructor: |
|
362 | # objects from constructor: | |
357 | query=Instance(ZMQStream) |
|
363 | query=Instance(ZMQStream) | |
358 | monitor=Instance(ZMQStream) |
|
364 | monitor=Instance(ZMQStream) | |
359 | notifier=Instance(ZMQStream) |
|
365 | notifier=Instance(ZMQStream) | |
360 | resubmit=Instance(ZMQStream) |
|
366 | resubmit=Instance(ZMQStream) | |
361 | heartmonitor=Instance(HeartMonitor) |
|
367 | heartmonitor=Instance(HeartMonitor) | |
362 | db=Instance(object) |
|
368 | db=Instance(object) | |
363 | client_info=Dict() |
|
369 | client_info=Dict() | |
364 | engine_info=Dict() |
|
370 | engine_info=Dict() | |
365 |
|
371 | |||
366 |
|
372 | |||
367 | def __init__(self, **kwargs): |
|
373 | def __init__(self, **kwargs): | |
368 | """ |
|
374 | """ | |
369 | # universal: |
|
375 | # universal: | |
370 | loop: IOLoop for creating future connections |
|
376 | loop: IOLoop for creating future connections | |
371 | session: streamsession for sending serialized data |
|
377 | session: streamsession for sending serialized data | |
372 | # engine: |
|
378 | # engine: | |
373 | queue: ZMQStream for monitoring queue messages |
|
379 | queue: ZMQStream for monitoring queue messages | |
374 | query: ZMQStream for engine+client registration and client requests |
|
380 | query: ZMQStream for engine+client registration and client requests | |
375 | heartbeat: HeartMonitor object for tracking engines |
|
381 | heartbeat: HeartMonitor object for tracking engines | |
376 | # extra: |
|
382 | # extra: | |
377 | db: ZMQStream for db connection (NotImplemented) |
|
383 | db: ZMQStream for db connection (NotImplemented) | |
378 | engine_info: zmq address/protocol dict for engine connections |
|
384 | engine_info: zmq address/protocol dict for engine connections | |
379 | client_info: zmq address/protocol dict for client connections |
|
385 | client_info: zmq address/protocol dict for client connections | |
380 | """ |
|
386 | """ | |
381 |
|
387 | |||
382 | super(Hub, self).__init__(**kwargs) |
|
388 | super(Hub, self).__init__(**kwargs) | |
383 | self.registration_timeout = max(5000, 2*self.heartmonitor.period) |
|
389 | self.registration_timeout = max(5000, 2*self.heartmonitor.period) | |
384 |
|
390 | |||
385 | # validate connection dicts: |
|
391 | # validate connection dicts: | |
386 | for k,v in self.client_info.iteritems(): |
|
392 | for k,v in self.client_info.iteritems(): | |
387 | if k == 'task': |
|
393 | if k == 'task_scheme': | |
388 |
|
|
394 | continue | |
389 | else: |
|
395 | else: | |
390 | util.validate_url_container(v) |
|
396 | util.validate_url_container(v) | |
391 | # util.validate_url_container(self.client_info) |
|
397 | # util.validate_url_container(self.client_info) | |
392 | util.validate_url_container(self.engine_info) |
|
398 | util.validate_url_container(self.engine_info) | |
393 |
|
399 | |||
394 | # register our callbacks |
|
400 | # register our callbacks | |
395 | self.query.on_recv(self.dispatch_query) |
|
401 | self.query.on_recv(self.dispatch_query) | |
396 | self.monitor.on_recv(self.dispatch_monitor_traffic) |
|
402 | self.monitor.on_recv(self.dispatch_monitor_traffic) | |
397 |
|
403 | |||
398 | self.heartmonitor.add_heart_failure_handler(self.handle_heart_failure) |
|
404 | self.heartmonitor.add_heart_failure_handler(self.handle_heart_failure) | |
399 | self.heartmonitor.add_new_heart_handler(self.handle_new_heart) |
|
405 | self.heartmonitor.add_new_heart_handler(self.handle_new_heart) | |
400 |
|
406 | |||
401 | self.monitor_handlers = {b'in' : self.save_queue_request, |
|
407 | self.monitor_handlers = {b'in' : self.save_queue_request, | |
402 | b'out': self.save_queue_result, |
|
408 | b'out': self.save_queue_result, | |
403 | b'intask': self.save_task_request, |
|
409 | b'intask': self.save_task_request, | |
404 | b'outtask': self.save_task_result, |
|
410 | b'outtask': self.save_task_result, | |
405 | b'tracktask': self.save_task_destination, |
|
411 | b'tracktask': self.save_task_destination, | |
406 | b'incontrol': _passer, |
|
412 | b'incontrol': _passer, | |
407 | b'outcontrol': _passer, |
|
413 | b'outcontrol': _passer, | |
408 | b'iopub': self.save_iopub_message, |
|
414 | b'iopub': self.save_iopub_message, | |
409 | } |
|
415 | } | |
410 |
|
416 | |||
411 | self.query_handlers = {'queue_request': self.queue_status, |
|
417 | self.query_handlers = {'queue_request': self.queue_status, | |
412 | 'result_request': self.get_results, |
|
418 | 'result_request': self.get_results, | |
413 | 'history_request': self.get_history, |
|
419 | 'history_request': self.get_history, | |
414 | 'db_request': self.db_query, |
|
420 | 'db_request': self.db_query, | |
415 | 'purge_request': self.purge_results, |
|
421 | 'purge_request': self.purge_results, | |
416 | 'load_request': self.check_load, |
|
422 | 'load_request': self.check_load, | |
417 | 'resubmit_request': self.resubmit_task, |
|
423 | 'resubmit_request': self.resubmit_task, | |
418 | 'shutdown_request': self.shutdown_request, |
|
424 | 'shutdown_request': self.shutdown_request, | |
419 | 'registration_request' : self.register_engine, |
|
425 | 'registration_request' : self.register_engine, | |
420 | 'unregistration_request' : self.unregister_engine, |
|
426 | 'unregistration_request' : self.unregister_engine, | |
421 | 'connection_request': self.connection_request, |
|
427 | 'connection_request': self.connection_request, | |
422 | } |
|
428 | } | |
423 |
|
429 | |||
424 | # ignore resubmit replies |
|
430 | # ignore resubmit replies | |
425 | self.resubmit.on_recv(lambda msg: None, copy=False) |
|
431 | self.resubmit.on_recv(lambda msg: None, copy=False) | |
426 |
|
432 | |||
427 | self.log.info("hub::created hub") |
|
433 | self.log.info("hub::created hub") | |
428 |
|
434 | |||
429 | @property |
|
435 | @property | |
430 | def _next_id(self): |
|
436 | def _next_id(self): | |
431 | """gemerate a new ID. |
|
437 | """gemerate a new ID. | |
432 |
|
438 | |||
433 | No longer reuse old ids, just count from 0.""" |
|
439 | No longer reuse old ids, just count from 0.""" | |
434 | newid = self._idcounter |
|
440 | newid = self._idcounter | |
435 | self._idcounter += 1 |
|
441 | self._idcounter += 1 | |
436 | return newid |
|
442 | return newid | |
437 | # newid = 0 |
|
443 | # newid = 0 | |
438 | # incoming = [id[0] for id in self.incoming_registrations.itervalues()] |
|
444 | # incoming = [id[0] for id in self.incoming_registrations.itervalues()] | |
439 | # # print newid, self.ids, self.incoming_registrations |
|
445 | # # print newid, self.ids, self.incoming_registrations | |
440 | # while newid in self.ids or newid in incoming: |
|
446 | # while newid in self.ids or newid in incoming: | |
441 | # newid += 1 |
|
447 | # newid += 1 | |
442 | # return newid |
|
448 | # return newid | |
443 |
|
449 | |||
444 | #----------------------------------------------------------------------------- |
|
450 | #----------------------------------------------------------------------------- | |
445 | # message validation |
|
451 | # message validation | |
446 | #----------------------------------------------------------------------------- |
|
452 | #----------------------------------------------------------------------------- | |
447 |
|
453 | |||
448 | def _validate_targets(self, targets): |
|
454 | def _validate_targets(self, targets): | |
449 | """turn any valid targets argument into a list of integer ids""" |
|
455 | """turn any valid targets argument into a list of integer ids""" | |
450 | if targets is None: |
|
456 | if targets is None: | |
451 | # default to all |
|
457 | # default to all | |
452 | return self.ids |
|
458 | return self.ids | |
453 |
|
459 | |||
454 | if isinstance(targets, (int,str,unicode)): |
|
460 | if isinstance(targets, (int,str,unicode)): | |
455 | # only one target specified |
|
461 | # only one target specified | |
456 | targets = [targets] |
|
462 | targets = [targets] | |
457 | _targets = [] |
|
463 | _targets = [] | |
458 | for t in targets: |
|
464 | for t in targets: | |
459 | # map raw identities to ids |
|
465 | # map raw identities to ids | |
460 | if isinstance(t, (str,unicode)): |
|
466 | if isinstance(t, (str,unicode)): | |
461 | t = self.by_ident.get(cast_bytes(t), t) |
|
467 | t = self.by_ident.get(cast_bytes(t), t) | |
462 | _targets.append(t) |
|
468 | _targets.append(t) | |
463 | targets = _targets |
|
469 | targets = _targets | |
464 | bad_targets = [ t for t in targets if t not in self.ids ] |
|
470 | bad_targets = [ t for t in targets if t not in self.ids ] | |
465 | if bad_targets: |
|
471 | if bad_targets: | |
466 | raise IndexError("No Such Engine: %r" % bad_targets) |
|
472 | raise IndexError("No Such Engine: %r" % bad_targets) | |
467 | if not targets: |
|
473 | if not targets: | |
468 | raise IndexError("No Engines Registered") |
|
474 | raise IndexError("No Engines Registered") | |
469 | return targets |
|
475 | return targets | |
470 |
|
476 | |||
471 | #----------------------------------------------------------------------------- |
|
477 | #----------------------------------------------------------------------------- | |
472 | # dispatch methods (1 per stream) |
|
478 | # dispatch methods (1 per stream) | |
473 | #----------------------------------------------------------------------------- |
|
479 | #----------------------------------------------------------------------------- | |
474 |
|
480 | |||
475 |
|
481 | |||
476 | @util.log_errors |
|
482 | @util.log_errors | |
477 | def dispatch_monitor_traffic(self, msg): |
|
483 | def dispatch_monitor_traffic(self, msg): | |
478 | """all ME and Task queue messages come through here, as well as |
|
484 | """all ME and Task queue messages come through here, as well as | |
479 | IOPub traffic.""" |
|
485 | IOPub traffic.""" | |
480 | self.log.debug("monitor traffic: %r", msg[0]) |
|
486 | self.log.debug("monitor traffic: %r", msg[0]) | |
481 | switch = msg[0] |
|
487 | switch = msg[0] | |
482 | try: |
|
488 | try: | |
483 | idents, msg = self.session.feed_identities(msg[1:]) |
|
489 | idents, msg = self.session.feed_identities(msg[1:]) | |
484 | except ValueError: |
|
490 | except ValueError: | |
485 | idents=[] |
|
491 | idents=[] | |
486 | if not idents: |
|
492 | if not idents: | |
487 | self.log.error("Monitor message without topic: %r", msg) |
|
493 | self.log.error("Monitor message without topic: %r", msg) | |
488 | return |
|
494 | return | |
489 | handler = self.monitor_handlers.get(switch, None) |
|
495 | handler = self.monitor_handlers.get(switch, None) | |
490 | if handler is not None: |
|
496 | if handler is not None: | |
491 | handler(idents, msg) |
|
497 | handler(idents, msg) | |
492 | else: |
|
498 | else: | |
493 | self.log.error("Unrecognized monitor topic: %r", switch) |
|
499 | self.log.error("Unrecognized monitor topic: %r", switch) | |
494 |
|
500 | |||
495 |
|
501 | |||
496 | @util.log_errors |
|
502 | @util.log_errors | |
497 | def dispatch_query(self, msg): |
|
503 | def dispatch_query(self, msg): | |
498 | """Route registration requests and queries from clients.""" |
|
504 | """Route registration requests and queries from clients.""" | |
499 | try: |
|
505 | try: | |
500 | idents, msg = self.session.feed_identities(msg) |
|
506 | idents, msg = self.session.feed_identities(msg) | |
501 | except ValueError: |
|
507 | except ValueError: | |
502 | idents = [] |
|
508 | idents = [] | |
503 | if not idents: |
|
509 | if not idents: | |
504 | self.log.error("Bad Query Message: %r", msg) |
|
510 | self.log.error("Bad Query Message: %r", msg) | |
505 | return |
|
511 | return | |
506 | client_id = idents[0] |
|
512 | client_id = idents[0] | |
507 | try: |
|
513 | try: | |
508 | msg = self.session.unserialize(msg, content=True) |
|
514 | msg = self.session.unserialize(msg, content=True) | |
509 | except Exception: |
|
515 | except Exception: | |
510 | content = error.wrap_exception() |
|
516 | content = error.wrap_exception() | |
511 | self.log.error("Bad Query Message: %r", msg, exc_info=True) |
|
517 | self.log.error("Bad Query Message: %r", msg, exc_info=True) | |
512 | self.session.send(self.query, "hub_error", ident=client_id, |
|
518 | self.session.send(self.query, "hub_error", ident=client_id, | |
513 | content=content) |
|
519 | content=content) | |
514 | return |
|
520 | return | |
515 | # print client_id, header, parent, content |
|
521 | # print client_id, header, parent, content | |
516 | #switch on message type: |
|
522 | #switch on message type: | |
517 | msg_type = msg['header']['msg_type'] |
|
523 | msg_type = msg['header']['msg_type'] | |
518 | self.log.info("client::client %r requested %r", client_id, msg_type) |
|
524 | self.log.info("client::client %r requested %r", client_id, msg_type) | |
519 | handler = self.query_handlers.get(msg_type, None) |
|
525 | handler = self.query_handlers.get(msg_type, None) | |
520 | try: |
|
526 | try: | |
521 | assert handler is not None, "Bad Message Type: %r" % msg_type |
|
527 | assert handler is not None, "Bad Message Type: %r" % msg_type | |
522 | except: |
|
528 | except: | |
523 | content = error.wrap_exception() |
|
529 | content = error.wrap_exception() | |
524 | self.log.error("Bad Message Type: %r", msg_type, exc_info=True) |
|
530 | self.log.error("Bad Message Type: %r", msg_type, exc_info=True) | |
525 | self.session.send(self.query, "hub_error", ident=client_id, |
|
531 | self.session.send(self.query, "hub_error", ident=client_id, | |
526 | content=content) |
|
532 | content=content) | |
527 | return |
|
533 | return | |
528 |
|
534 | |||
529 | else: |
|
535 | else: | |
530 | handler(idents, msg) |
|
536 | handler(idents, msg) | |
531 |
|
537 | |||
532 | def dispatch_db(self, msg): |
|
538 | def dispatch_db(self, msg): | |
533 | """""" |
|
539 | """""" | |
534 | raise NotImplementedError |
|
540 | raise NotImplementedError | |
535 |
|
541 | |||
536 | #--------------------------------------------------------------------------- |
|
542 | #--------------------------------------------------------------------------- | |
537 | # handler methods (1 per event) |
|
543 | # handler methods (1 per event) | |
538 | #--------------------------------------------------------------------------- |
|
544 | #--------------------------------------------------------------------------- | |
539 |
|
545 | |||
540 | #----------------------- Heartbeat -------------------------------------- |
|
546 | #----------------------- Heartbeat -------------------------------------- | |
541 |
|
547 | |||
542 | def handle_new_heart(self, heart): |
|
548 | def handle_new_heart(self, heart): | |
543 | """handler to attach to heartbeater. |
|
549 | """handler to attach to heartbeater. | |
544 | Called when a new heart starts to beat. |
|
550 | Called when a new heart starts to beat. | |
545 | Triggers completion of registration.""" |
|
551 | Triggers completion of registration.""" | |
546 | self.log.debug("heartbeat::handle_new_heart(%r)", heart) |
|
552 | self.log.debug("heartbeat::handle_new_heart(%r)", heart) | |
547 | if heart not in self.incoming_registrations: |
|
553 | if heart not in self.incoming_registrations: | |
548 | self.log.info("heartbeat::ignoring new heart: %r", heart) |
|
554 | self.log.info("heartbeat::ignoring new heart: %r", heart) | |
549 | else: |
|
555 | else: | |
550 | self.finish_registration(heart) |
|
556 | self.finish_registration(heart) | |
551 |
|
557 | |||
552 |
|
558 | |||
553 | def handle_heart_failure(self, heart): |
|
559 | def handle_heart_failure(self, heart): | |
554 | """handler to attach to heartbeater. |
|
560 | """handler to attach to heartbeater. | |
555 | called when a previously registered heart fails to respond to beat request. |
|
561 | called when a previously registered heart fails to respond to beat request. | |
556 | triggers unregistration""" |
|
562 | triggers unregistration""" | |
557 | self.log.debug("heartbeat::handle_heart_failure(%r)", heart) |
|
563 | self.log.debug("heartbeat::handle_heart_failure(%r)", heart) | |
558 | eid = self.hearts.get(heart, None) |
|
564 | eid = self.hearts.get(heart, None) | |
559 | queue = self.engines[eid].queue |
|
565 | queue = self.engines[eid].queue | |
560 | if eid is None or self.keytable[eid] in self.dead_engines: |
|
566 | if eid is None or self.keytable[eid] in self.dead_engines: | |
561 | self.log.info("heartbeat::ignoring heart failure %r (not an engine or already dead)", heart) |
|
567 | self.log.info("heartbeat::ignoring heart failure %r (not an engine or already dead)", heart) | |
562 | else: |
|
568 | else: | |
563 | self.unregister_engine(heart, dict(content=dict(id=eid, queue=queue))) |
|
569 | self.unregister_engine(heart, dict(content=dict(id=eid, queue=queue))) | |
564 |
|
570 | |||
565 | #----------------------- MUX Queue Traffic ------------------------------ |
|
571 | #----------------------- MUX Queue Traffic ------------------------------ | |
566 |
|
572 | |||
567 | def save_queue_request(self, idents, msg): |
|
573 | def save_queue_request(self, idents, msg): | |
568 | if len(idents) < 2: |
|
574 | if len(idents) < 2: | |
569 | self.log.error("invalid identity prefix: %r", idents) |
|
575 | self.log.error("invalid identity prefix: %r", idents) | |
570 | return |
|
576 | return | |
571 | queue_id, client_id = idents[:2] |
|
577 | queue_id, client_id = idents[:2] | |
572 | try: |
|
578 | try: | |
573 | msg = self.session.unserialize(msg) |
|
579 | msg = self.session.unserialize(msg) | |
574 | except Exception: |
|
580 | except Exception: | |
575 | self.log.error("queue::client %r sent invalid message to %r: %r", client_id, queue_id, msg, exc_info=True) |
|
581 | self.log.error("queue::client %r sent invalid message to %r: %r", client_id, queue_id, msg, exc_info=True) | |
576 | return |
|
582 | return | |
577 |
|
583 | |||
578 | eid = self.by_ident.get(queue_id, None) |
|
584 | eid = self.by_ident.get(queue_id, None) | |
579 | if eid is None: |
|
585 | if eid is None: | |
580 | self.log.error("queue::target %r not registered", queue_id) |
|
586 | self.log.error("queue::target %r not registered", queue_id) | |
581 | self.log.debug("queue:: valid are: %r", self.by_ident.keys()) |
|
587 | self.log.debug("queue:: valid are: %r", self.by_ident.keys()) | |
582 | return |
|
588 | return | |
583 | record = init_record(msg) |
|
589 | record = init_record(msg) | |
584 | msg_id = record['msg_id'] |
|
590 | msg_id = record['msg_id'] | |
585 | self.log.info("queue::client %r submitted request %r to %s", client_id, msg_id, eid) |
|
591 | self.log.info("queue::client %r submitted request %r to %s", client_id, msg_id, eid) | |
586 | # Unicode in records |
|
592 | # Unicode in records | |
587 | record['engine_uuid'] = queue_id.decode('ascii') |
|
593 | record['engine_uuid'] = queue_id.decode('ascii') | |
588 | record['client_uuid'] = msg['header']['session'] |
|
594 | record['client_uuid'] = msg['header']['session'] | |
589 | record['queue'] = 'mux' |
|
595 | record['queue'] = 'mux' | |
590 |
|
596 | |||
591 | try: |
|
597 | try: | |
592 | # it's posible iopub arrived first: |
|
598 | # it's posible iopub arrived first: | |
593 | existing = self.db.get_record(msg_id) |
|
599 | existing = self.db.get_record(msg_id) | |
594 | for key,evalue in existing.iteritems(): |
|
600 | for key,evalue in existing.iteritems(): | |
595 | rvalue = record.get(key, None) |
|
601 | rvalue = record.get(key, None) | |
596 | if evalue and rvalue and evalue != rvalue: |
|
602 | if evalue and rvalue and evalue != rvalue: | |
597 | self.log.warn("conflicting initial state for record: %r:%r <%r> %r", msg_id, rvalue, key, evalue) |
|
603 | self.log.warn("conflicting initial state for record: %r:%r <%r> %r", msg_id, rvalue, key, evalue) | |
598 | elif evalue and not rvalue: |
|
604 | elif evalue and not rvalue: | |
599 | record[key] = evalue |
|
605 | record[key] = evalue | |
600 | try: |
|
606 | try: | |
601 | self.db.update_record(msg_id, record) |
|
607 | self.db.update_record(msg_id, record) | |
602 | except Exception: |
|
608 | except Exception: | |
603 | self.log.error("DB Error updating record %r", msg_id, exc_info=True) |
|
609 | self.log.error("DB Error updating record %r", msg_id, exc_info=True) | |
604 | except KeyError: |
|
610 | except KeyError: | |
605 | try: |
|
611 | try: | |
606 | self.db.add_record(msg_id, record) |
|
612 | self.db.add_record(msg_id, record) | |
607 | except Exception: |
|
613 | except Exception: | |
608 | self.log.error("DB Error adding record %r", msg_id, exc_info=True) |
|
614 | self.log.error("DB Error adding record %r", msg_id, exc_info=True) | |
609 |
|
615 | |||
610 |
|
616 | |||
611 | self.pending.add(msg_id) |
|
617 | self.pending.add(msg_id) | |
612 | self.queues[eid].append(msg_id) |
|
618 | self.queues[eid].append(msg_id) | |
613 |
|
619 | |||
614 | def save_queue_result(self, idents, msg): |
|
620 | def save_queue_result(self, idents, msg): | |
615 | if len(idents) < 2: |
|
621 | if len(idents) < 2: | |
616 | self.log.error("invalid identity prefix: %r", idents) |
|
622 | self.log.error("invalid identity prefix: %r", idents) | |
617 | return |
|
623 | return | |
618 |
|
624 | |||
619 | client_id, queue_id = idents[:2] |
|
625 | client_id, queue_id = idents[:2] | |
620 | try: |
|
626 | try: | |
621 | msg = self.session.unserialize(msg) |
|
627 | msg = self.session.unserialize(msg) | |
622 | except Exception: |
|
628 | except Exception: | |
623 | self.log.error("queue::engine %r sent invalid message to %r: %r", |
|
629 | self.log.error("queue::engine %r sent invalid message to %r: %r", | |
624 | queue_id, client_id, msg, exc_info=True) |
|
630 | queue_id, client_id, msg, exc_info=True) | |
625 | return |
|
631 | return | |
626 |
|
632 | |||
627 | eid = self.by_ident.get(queue_id, None) |
|
633 | eid = self.by_ident.get(queue_id, None) | |
628 | if eid is None: |
|
634 | if eid is None: | |
629 | self.log.error("queue::unknown engine %r is sending a reply: ", queue_id) |
|
635 | self.log.error("queue::unknown engine %r is sending a reply: ", queue_id) | |
630 | return |
|
636 | return | |
631 |
|
637 | |||
632 | parent = msg['parent_header'] |
|
638 | parent = msg['parent_header'] | |
633 | if not parent: |
|
639 | if not parent: | |
634 | return |
|
640 | return | |
635 | msg_id = parent['msg_id'] |
|
641 | msg_id = parent['msg_id'] | |
636 | if msg_id in self.pending: |
|
642 | if msg_id in self.pending: | |
637 | self.pending.remove(msg_id) |
|
643 | self.pending.remove(msg_id) | |
638 | self.all_completed.add(msg_id) |
|
644 | self.all_completed.add(msg_id) | |
639 | self.queues[eid].remove(msg_id) |
|
645 | self.queues[eid].remove(msg_id) | |
640 | self.completed[eid].append(msg_id) |
|
646 | self.completed[eid].append(msg_id) | |
641 | self.log.info("queue::request %r completed on %s", msg_id, eid) |
|
647 | self.log.info("queue::request %r completed on %s", msg_id, eid) | |
642 | elif msg_id not in self.all_completed: |
|
648 | elif msg_id not in self.all_completed: | |
643 | # it could be a result from a dead engine that died before delivering the |
|
649 | # it could be a result from a dead engine that died before delivering the | |
644 | # result |
|
650 | # result | |
645 | self.log.warn("queue:: unknown msg finished %r", msg_id) |
|
651 | self.log.warn("queue:: unknown msg finished %r", msg_id) | |
646 | return |
|
652 | return | |
647 | # update record anyway, because the unregistration could have been premature |
|
653 | # update record anyway, because the unregistration could have been premature | |
648 | rheader = msg['header'] |
|
654 | rheader = msg['header'] | |
649 | completed = rheader['date'] |
|
655 | completed = rheader['date'] | |
650 | started = rheader.get('started', None) |
|
656 | started = rheader.get('started', None) | |
651 | result = { |
|
657 | result = { | |
652 | 'result_header' : rheader, |
|
658 | 'result_header' : rheader, | |
653 | 'result_content': msg['content'], |
|
659 | 'result_content': msg['content'], | |
654 | 'received': datetime.now(), |
|
660 | 'received': datetime.now(), | |
655 | 'started' : started, |
|
661 | 'started' : started, | |
656 | 'completed' : completed |
|
662 | 'completed' : completed | |
657 | } |
|
663 | } | |
658 |
|
664 | |||
659 | result['result_buffers'] = msg['buffers'] |
|
665 | result['result_buffers'] = msg['buffers'] | |
660 | try: |
|
666 | try: | |
661 | self.db.update_record(msg_id, result) |
|
667 | self.db.update_record(msg_id, result) | |
662 | except Exception: |
|
668 | except Exception: | |
663 | self.log.error("DB Error updating record %r", msg_id, exc_info=True) |
|
669 | self.log.error("DB Error updating record %r", msg_id, exc_info=True) | |
664 |
|
670 | |||
665 |
|
671 | |||
666 | #--------------------- Task Queue Traffic ------------------------------ |
|
672 | #--------------------- Task Queue Traffic ------------------------------ | |
667 |
|
673 | |||
668 | def save_task_request(self, idents, msg): |
|
674 | def save_task_request(self, idents, msg): | |
669 | """Save the submission of a task.""" |
|
675 | """Save the submission of a task.""" | |
670 | client_id = idents[0] |
|
676 | client_id = idents[0] | |
671 |
|
677 | |||
672 | try: |
|
678 | try: | |
673 | msg = self.session.unserialize(msg) |
|
679 | msg = self.session.unserialize(msg) | |
674 | except Exception: |
|
680 | except Exception: | |
675 | self.log.error("task::client %r sent invalid task message: %r", |
|
681 | self.log.error("task::client %r sent invalid task message: %r", | |
676 | client_id, msg, exc_info=True) |
|
682 | client_id, msg, exc_info=True) | |
677 | return |
|
683 | return | |
678 | record = init_record(msg) |
|
684 | record = init_record(msg) | |
679 |
|
685 | |||
680 | record['client_uuid'] = msg['header']['session'] |
|
686 | record['client_uuid'] = msg['header']['session'] | |
681 | record['queue'] = 'task' |
|
687 | record['queue'] = 'task' | |
682 | header = msg['header'] |
|
688 | header = msg['header'] | |
683 | msg_id = header['msg_id'] |
|
689 | msg_id = header['msg_id'] | |
684 | self.pending.add(msg_id) |
|
690 | self.pending.add(msg_id) | |
685 | self.unassigned.add(msg_id) |
|
691 | self.unassigned.add(msg_id) | |
686 | try: |
|
692 | try: | |
687 | # it's posible iopub arrived first: |
|
693 | # it's posible iopub arrived first: | |
688 | existing = self.db.get_record(msg_id) |
|
694 | existing = self.db.get_record(msg_id) | |
689 | if existing['resubmitted']: |
|
695 | if existing['resubmitted']: | |
690 | for key in ('submitted', 'client_uuid', 'buffers'): |
|
696 | for key in ('submitted', 'client_uuid', 'buffers'): | |
691 | # don't clobber these keys on resubmit |
|
697 | # don't clobber these keys on resubmit | |
692 | # submitted and client_uuid should be different |
|
698 | # submitted and client_uuid should be different | |
693 | # and buffers might be big, and shouldn't have changed |
|
699 | # and buffers might be big, and shouldn't have changed | |
694 | record.pop(key) |
|
700 | record.pop(key) | |
695 | # still check content,header which should not change |
|
701 | # still check content,header which should not change | |
696 | # but are not expensive to compare as buffers |
|
702 | # but are not expensive to compare as buffers | |
697 |
|
703 | |||
698 | for key,evalue in existing.iteritems(): |
|
704 | for key,evalue in existing.iteritems(): | |
699 | if key.endswith('buffers'): |
|
705 | if key.endswith('buffers'): | |
700 | # don't compare buffers |
|
706 | # don't compare buffers | |
701 | continue |
|
707 | continue | |
702 | rvalue = record.get(key, None) |
|
708 | rvalue = record.get(key, None) | |
703 | if evalue and rvalue and evalue != rvalue: |
|
709 | if evalue and rvalue and evalue != rvalue: | |
704 | self.log.warn("conflicting initial state for record: %r:%r <%r> %r", msg_id, rvalue, key, evalue) |
|
710 | self.log.warn("conflicting initial state for record: %r:%r <%r> %r", msg_id, rvalue, key, evalue) | |
705 | elif evalue and not rvalue: |
|
711 | elif evalue and not rvalue: | |
706 | record[key] = evalue |
|
712 | record[key] = evalue | |
707 | try: |
|
713 | try: | |
708 | self.db.update_record(msg_id, record) |
|
714 | self.db.update_record(msg_id, record) | |
709 | except Exception: |
|
715 | except Exception: | |
710 | self.log.error("DB Error updating record %r", msg_id, exc_info=True) |
|
716 | self.log.error("DB Error updating record %r", msg_id, exc_info=True) | |
711 | except KeyError: |
|
717 | except KeyError: | |
712 | try: |
|
718 | try: | |
713 | self.db.add_record(msg_id, record) |
|
719 | self.db.add_record(msg_id, record) | |
714 | except Exception: |
|
720 | except Exception: | |
715 | self.log.error("DB Error adding record %r", msg_id, exc_info=True) |
|
721 | self.log.error("DB Error adding record %r", msg_id, exc_info=True) | |
716 | except Exception: |
|
722 | except Exception: | |
717 | self.log.error("DB Error saving task request %r", msg_id, exc_info=True) |
|
723 | self.log.error("DB Error saving task request %r", msg_id, exc_info=True) | |
718 |
|
724 | |||
719 | def save_task_result(self, idents, msg): |
|
725 | def save_task_result(self, idents, msg): | |
720 | """save the result of a completed task.""" |
|
726 | """save the result of a completed task.""" | |
721 | client_id = idents[0] |
|
727 | client_id = idents[0] | |
722 | try: |
|
728 | try: | |
723 | msg = self.session.unserialize(msg) |
|
729 | msg = self.session.unserialize(msg) | |
724 | except Exception: |
|
730 | except Exception: | |
725 | self.log.error("task::invalid task result message send to %r: %r", |
|
731 | self.log.error("task::invalid task result message send to %r: %r", | |
726 | client_id, msg, exc_info=True) |
|
732 | client_id, msg, exc_info=True) | |
727 | return |
|
733 | return | |
728 |
|
734 | |||
729 | parent = msg['parent_header'] |
|
735 | parent = msg['parent_header'] | |
730 | if not parent: |
|
736 | if not parent: | |
731 | # print msg |
|
737 | # print msg | |
732 | self.log.warn("Task %r had no parent!", msg) |
|
738 | self.log.warn("Task %r had no parent!", msg) | |
733 | return |
|
739 | return | |
734 | msg_id = parent['msg_id'] |
|
740 | msg_id = parent['msg_id'] | |
735 | if msg_id in self.unassigned: |
|
741 | if msg_id in self.unassigned: | |
736 | self.unassigned.remove(msg_id) |
|
742 | self.unassigned.remove(msg_id) | |
737 |
|
743 | |||
738 | header = msg['header'] |
|
744 | header = msg['header'] | |
739 | engine_uuid = header.get('engine', u'') |
|
745 | engine_uuid = header.get('engine', u'') | |
740 | eid = self.by_ident.get(cast_bytes(engine_uuid), None) |
|
746 | eid = self.by_ident.get(cast_bytes(engine_uuid), None) | |
741 |
|
747 | |||
742 | status = header.get('status', None) |
|
748 | status = header.get('status', None) | |
743 |
|
749 | |||
744 | if msg_id in self.pending: |
|
750 | if msg_id in self.pending: | |
745 | self.log.info("task::task %r finished on %s", msg_id, eid) |
|
751 | self.log.info("task::task %r finished on %s", msg_id, eid) | |
746 | self.pending.remove(msg_id) |
|
752 | self.pending.remove(msg_id) | |
747 | self.all_completed.add(msg_id) |
|
753 | self.all_completed.add(msg_id) | |
748 | if eid is not None: |
|
754 | if eid is not None: | |
749 | if status != 'aborted': |
|
755 | if status != 'aborted': | |
750 | self.completed[eid].append(msg_id) |
|
756 | self.completed[eid].append(msg_id) | |
751 | if msg_id in self.tasks[eid]: |
|
757 | if msg_id in self.tasks[eid]: | |
752 | self.tasks[eid].remove(msg_id) |
|
758 | self.tasks[eid].remove(msg_id) | |
753 | completed = header['date'] |
|
759 | completed = header['date'] | |
754 | started = header.get('started', None) |
|
760 | started = header.get('started', None) | |
755 | result = { |
|
761 | result = { | |
756 | 'result_header' : header, |
|
762 | 'result_header' : header, | |
757 | 'result_content': msg['content'], |
|
763 | 'result_content': msg['content'], | |
758 | 'started' : started, |
|
764 | 'started' : started, | |
759 | 'completed' : completed, |
|
765 | 'completed' : completed, | |
760 | 'received' : datetime.now(), |
|
766 | 'received' : datetime.now(), | |
761 | 'engine_uuid': engine_uuid, |
|
767 | 'engine_uuid': engine_uuid, | |
762 | } |
|
768 | } | |
763 |
|
769 | |||
764 | result['result_buffers'] = msg['buffers'] |
|
770 | result['result_buffers'] = msg['buffers'] | |
765 | try: |
|
771 | try: | |
766 | self.db.update_record(msg_id, result) |
|
772 | self.db.update_record(msg_id, result) | |
767 | except Exception: |
|
773 | except Exception: | |
768 | self.log.error("DB Error saving task request %r", msg_id, exc_info=True) |
|
774 | self.log.error("DB Error saving task request %r", msg_id, exc_info=True) | |
769 |
|
775 | |||
770 | else: |
|
776 | else: | |
771 | self.log.debug("task::unknown task %r finished", msg_id) |
|
777 | self.log.debug("task::unknown task %r finished", msg_id) | |
772 |
|
778 | |||
773 | def save_task_destination(self, idents, msg): |
|
779 | def save_task_destination(self, idents, msg): | |
774 | try: |
|
780 | try: | |
775 | msg = self.session.unserialize(msg, content=True) |
|
781 | msg = self.session.unserialize(msg, content=True) | |
776 | except Exception: |
|
782 | except Exception: | |
777 | self.log.error("task::invalid task tracking message", exc_info=True) |
|
783 | self.log.error("task::invalid task tracking message", exc_info=True) | |
778 | return |
|
784 | return | |
779 | content = msg['content'] |
|
785 | content = msg['content'] | |
780 | # print (content) |
|
786 | # print (content) | |
781 | msg_id = content['msg_id'] |
|
787 | msg_id = content['msg_id'] | |
782 | engine_uuid = content['engine_id'] |
|
788 | engine_uuid = content['engine_id'] | |
783 | eid = self.by_ident[cast_bytes(engine_uuid)] |
|
789 | eid = self.by_ident[cast_bytes(engine_uuid)] | |
784 |
|
790 | |||
785 | self.log.info("task::task %r arrived on %r", msg_id, eid) |
|
791 | self.log.info("task::task %r arrived on %r", msg_id, eid) | |
786 | if msg_id in self.unassigned: |
|
792 | if msg_id in self.unassigned: | |
787 | self.unassigned.remove(msg_id) |
|
793 | self.unassigned.remove(msg_id) | |
788 | # else: |
|
794 | # else: | |
789 | # self.log.debug("task::task %r not listed as MIA?!"%(msg_id)) |
|
795 | # self.log.debug("task::task %r not listed as MIA?!"%(msg_id)) | |
790 |
|
796 | |||
791 | self.tasks[eid].append(msg_id) |
|
797 | self.tasks[eid].append(msg_id) | |
792 | # self.pending[msg_id][1].update(received=datetime.now(),engine=(eid,engine_uuid)) |
|
798 | # self.pending[msg_id][1].update(received=datetime.now(),engine=(eid,engine_uuid)) | |
793 | try: |
|
799 | try: | |
794 | self.db.update_record(msg_id, dict(engine_uuid=engine_uuid)) |
|
800 | self.db.update_record(msg_id, dict(engine_uuid=engine_uuid)) | |
795 | except Exception: |
|
801 | except Exception: | |
796 | self.log.error("DB Error saving task destination %r", msg_id, exc_info=True) |
|
802 | self.log.error("DB Error saving task destination %r", msg_id, exc_info=True) | |
797 |
|
803 | |||
798 |
|
804 | |||
799 | def mia_task_request(self, idents, msg): |
|
805 | def mia_task_request(self, idents, msg): | |
800 | raise NotImplementedError |
|
806 | raise NotImplementedError | |
801 | client_id = idents[0] |
|
807 | client_id = idents[0] | |
802 | # content = dict(mia=self.mia,status='ok') |
|
808 | # content = dict(mia=self.mia,status='ok') | |
803 | # self.session.send('mia_reply', content=content, idents=client_id) |
|
809 | # self.session.send('mia_reply', content=content, idents=client_id) | |
804 |
|
810 | |||
805 |
|
811 | |||
806 | #--------------------- IOPub Traffic ------------------------------ |
|
812 | #--------------------- IOPub Traffic ------------------------------ | |
807 |
|
813 | |||
808 | def save_iopub_message(self, topics, msg): |
|
814 | def save_iopub_message(self, topics, msg): | |
809 | """save an iopub message into the db""" |
|
815 | """save an iopub message into the db""" | |
810 | # print (topics) |
|
816 | # print (topics) | |
811 | try: |
|
817 | try: | |
812 | msg = self.session.unserialize(msg, content=True) |
|
818 | msg = self.session.unserialize(msg, content=True) | |
813 | except Exception: |
|
819 | except Exception: | |
814 | self.log.error("iopub::invalid IOPub message", exc_info=True) |
|
820 | self.log.error("iopub::invalid IOPub message", exc_info=True) | |
815 | return |
|
821 | return | |
816 |
|
822 | |||
817 | parent = msg['parent_header'] |
|
823 | parent = msg['parent_header'] | |
818 | if not parent: |
|
824 | if not parent: | |
819 | self.log.warn("iopub::IOPub message lacks parent: %r", msg) |
|
825 | self.log.warn("iopub::IOPub message lacks parent: %r", msg) | |
820 | return |
|
826 | return | |
821 | msg_id = parent['msg_id'] |
|
827 | msg_id = parent['msg_id'] | |
822 | msg_type = msg['header']['msg_type'] |
|
828 | msg_type = msg['header']['msg_type'] | |
823 | content = msg['content'] |
|
829 | content = msg['content'] | |
824 |
|
830 | |||
825 | # ensure msg_id is in db |
|
831 | # ensure msg_id is in db | |
826 | try: |
|
832 | try: | |
827 | rec = self.db.get_record(msg_id) |
|
833 | rec = self.db.get_record(msg_id) | |
828 | except KeyError: |
|
834 | except KeyError: | |
829 | rec = empty_record() |
|
835 | rec = empty_record() | |
830 | rec['msg_id'] = msg_id |
|
836 | rec['msg_id'] = msg_id | |
831 | self.db.add_record(msg_id, rec) |
|
837 | self.db.add_record(msg_id, rec) | |
832 | # stream |
|
838 | # stream | |
833 | d = {} |
|
839 | d = {} | |
834 | if msg_type == 'stream': |
|
840 | if msg_type == 'stream': | |
835 | name = content['name'] |
|
841 | name = content['name'] | |
836 | s = rec[name] or '' |
|
842 | s = rec[name] or '' | |
837 | d[name] = s + content['data'] |
|
843 | d[name] = s + content['data'] | |
838 |
|
844 | |||
839 | elif msg_type == 'pyerr': |
|
845 | elif msg_type == 'pyerr': | |
840 | d['pyerr'] = content |
|
846 | d['pyerr'] = content | |
841 | elif msg_type == 'pyin': |
|
847 | elif msg_type == 'pyin': | |
842 | d['pyin'] = content['code'] |
|
848 | d['pyin'] = content['code'] | |
843 | elif msg_type in ('display_data', 'pyout'): |
|
849 | elif msg_type in ('display_data', 'pyout'): | |
844 | d[msg_type] = content |
|
850 | d[msg_type] = content | |
845 | elif msg_type == 'status': |
|
851 | elif msg_type == 'status': | |
846 | pass |
|
852 | pass | |
847 | else: |
|
853 | else: | |
848 | self.log.warn("unhandled iopub msg_type: %r", msg_type) |
|
854 | self.log.warn("unhandled iopub msg_type: %r", msg_type) | |
849 |
|
855 | |||
850 | if not d: |
|
856 | if not d: | |
851 | return |
|
857 | return | |
852 |
|
858 | |||
853 | try: |
|
859 | try: | |
854 | self.db.update_record(msg_id, d) |
|
860 | self.db.update_record(msg_id, d) | |
855 | except Exception: |
|
861 | except Exception: | |
856 | self.log.error("DB Error saving iopub message %r", msg_id, exc_info=True) |
|
862 | self.log.error("DB Error saving iopub message %r", msg_id, exc_info=True) | |
857 |
|
863 | |||
858 |
|
864 | |||
859 |
|
865 | |||
860 | #------------------------------------------------------------------------- |
|
866 | #------------------------------------------------------------------------- | |
861 | # Registration requests |
|
867 | # Registration requests | |
862 | #------------------------------------------------------------------------- |
|
868 | #------------------------------------------------------------------------- | |
863 |
|
869 | |||
864 | def connection_request(self, client_id, msg): |
|
870 | def connection_request(self, client_id, msg): | |
865 | """Reply with connection addresses for clients.""" |
|
871 | """Reply with connection addresses for clients.""" | |
866 | self.log.info("client::client %r connected", client_id) |
|
872 | self.log.info("client::client %r connected", client_id) | |
867 | content = dict(status='ok') |
|
873 | content = dict(status='ok') | |
868 | content.update(self.client_info) |
|
|||
869 | jsonable = {} |
|
874 | jsonable = {} | |
870 | for k,v in self.keytable.iteritems(): |
|
875 | for k,v in self.keytable.iteritems(): | |
871 | if v not in self.dead_engines: |
|
876 | if v not in self.dead_engines: | |
872 | jsonable[str(k)] = v.decode('ascii') |
|
877 | jsonable[str(k)] = v.decode('ascii') | |
873 | content['engines'] = jsonable |
|
878 | content['engines'] = jsonable | |
874 | self.session.send(self.query, 'connection_reply', content, parent=msg, ident=client_id) |
|
879 | self.session.send(self.query, 'connection_reply', content, parent=msg, ident=client_id) | |
875 |
|
880 | |||
876 | def register_engine(self, reg, msg): |
|
881 | def register_engine(self, reg, msg): | |
877 | """Register a new engine.""" |
|
882 | """Register a new engine.""" | |
878 | content = msg['content'] |
|
883 | content = msg['content'] | |
879 | try: |
|
884 | try: | |
880 | queue = cast_bytes(content['queue']) |
|
885 | queue = cast_bytes(content['queue']) | |
881 | except KeyError: |
|
886 | except KeyError: | |
882 | self.log.error("registration::queue not specified", exc_info=True) |
|
887 | self.log.error("registration::queue not specified", exc_info=True) | |
883 | return |
|
888 | return | |
884 | heart = content.get('heartbeat', None) |
|
889 | heart = content.get('heartbeat', None) | |
885 | if heart: |
|
890 | if heart: | |
886 | heart = cast_bytes(heart) |
|
891 | heart = cast_bytes(heart) | |
887 | """register a new engine, and create the socket(s) necessary""" |
|
892 | """register a new engine, and create the socket(s) necessary""" | |
888 | eid = self._next_id |
|
893 | eid = self._next_id | |
889 | # print (eid, queue, reg, heart) |
|
894 | # print (eid, queue, reg, heart) | |
890 |
|
895 | |||
891 | self.log.debug("registration::register_engine(%i, %r, %r, %r)", eid, queue, reg, heart) |
|
896 | self.log.debug("registration::register_engine(%i, %r, %r, %r)", eid, queue, reg, heart) | |
892 |
|
897 | |||
893 | content = dict(id=eid,status='ok') |
|
898 | content = dict(id=eid,status='ok') | |
894 | content.update(self.engine_info) |
|
|||
895 | # check if requesting available IDs: |
|
899 | # check if requesting available IDs: | |
896 | if queue in self.by_ident: |
|
900 | if queue in self.by_ident: | |
897 | try: |
|
901 | try: | |
898 | raise KeyError("queue_id %r in use" % queue) |
|
902 | raise KeyError("queue_id %r in use" % queue) | |
899 | except: |
|
903 | except: | |
900 | content = error.wrap_exception() |
|
904 | content = error.wrap_exception() | |
901 | self.log.error("queue_id %r in use", queue, exc_info=True) |
|
905 | self.log.error("queue_id %r in use", queue, exc_info=True) | |
902 | elif heart in self.hearts: # need to check unique hearts? |
|
906 | elif heart in self.hearts: # need to check unique hearts? | |
903 | try: |
|
907 | try: | |
904 | raise KeyError("heart_id %r in use" % heart) |
|
908 | raise KeyError("heart_id %r in use" % heart) | |
905 | except: |
|
909 | except: | |
906 | self.log.error("heart_id %r in use", heart, exc_info=True) |
|
910 | self.log.error("heart_id %r in use", heart, exc_info=True) | |
907 | content = error.wrap_exception() |
|
911 | content = error.wrap_exception() | |
908 | else: |
|
912 | else: | |
909 | for h, pack in self.incoming_registrations.iteritems(): |
|
913 | for h, pack in self.incoming_registrations.iteritems(): | |
910 | if heart == h: |
|
914 | if heart == h: | |
911 | try: |
|
915 | try: | |
912 | raise KeyError("heart_id %r in use" % heart) |
|
916 | raise KeyError("heart_id %r in use" % heart) | |
913 | except: |
|
917 | except: | |
914 | self.log.error("heart_id %r in use", heart, exc_info=True) |
|
918 | self.log.error("heart_id %r in use", heart, exc_info=True) | |
915 | content = error.wrap_exception() |
|
919 | content = error.wrap_exception() | |
916 | break |
|
920 | break | |
917 | elif queue == pack[1]: |
|
921 | elif queue == pack[1]: | |
918 | try: |
|
922 | try: | |
919 | raise KeyError("queue_id %r in use" % queue) |
|
923 | raise KeyError("queue_id %r in use" % queue) | |
920 | except: |
|
924 | except: | |
921 | self.log.error("queue_id %r in use", queue, exc_info=True) |
|
925 | self.log.error("queue_id %r in use", queue, exc_info=True) | |
922 | content = error.wrap_exception() |
|
926 | content = error.wrap_exception() | |
923 | break |
|
927 | break | |
924 |
|
928 | |||
925 | msg = self.session.send(self.query, "registration_reply", |
|
929 | msg = self.session.send(self.query, "registration_reply", | |
926 | content=content, |
|
930 | content=content, | |
927 | ident=reg) |
|
931 | ident=reg) | |
928 |
|
932 | |||
929 | if content['status'] == 'ok': |
|
933 | if content['status'] == 'ok': | |
930 | if heart in self.heartmonitor.hearts: |
|
934 | if heart in self.heartmonitor.hearts: | |
931 | # already beating |
|
935 | # already beating | |
932 | self.incoming_registrations[heart] = (eid,queue,reg[0],None) |
|
936 | self.incoming_registrations[heart] = (eid,queue,reg[0],None) | |
933 | self.finish_registration(heart) |
|
937 | self.finish_registration(heart) | |
934 | else: |
|
938 | else: | |
935 | purge = lambda : self._purge_stalled_registration(heart) |
|
939 | purge = lambda : self._purge_stalled_registration(heart) | |
936 | dc = ioloop.DelayedCallback(purge, self.registration_timeout, self.loop) |
|
940 | dc = ioloop.DelayedCallback(purge, self.registration_timeout, self.loop) | |
937 | dc.start() |
|
941 | dc.start() | |
938 | self.incoming_registrations[heart] = (eid,queue,reg[0],dc) |
|
942 | self.incoming_registrations[heart] = (eid,queue,reg[0],dc) | |
939 | else: |
|
943 | else: | |
940 | self.log.error("registration::registration %i failed: %r", eid, content['evalue']) |
|
944 | self.log.error("registration::registration %i failed: %r", eid, content['evalue']) | |
941 | return eid |
|
945 | return eid | |
942 |
|
946 | |||
943 | def unregister_engine(self, ident, msg): |
|
947 | def unregister_engine(self, ident, msg): | |
944 | """Unregister an engine that explicitly requested to leave.""" |
|
948 | """Unregister an engine that explicitly requested to leave.""" | |
945 | try: |
|
949 | try: | |
946 | eid = msg['content']['id'] |
|
950 | eid = msg['content']['id'] | |
947 | except: |
|
951 | except: | |
948 | self.log.error("registration::bad engine id for unregistration: %r", ident, exc_info=True) |
|
952 | self.log.error("registration::bad engine id for unregistration: %r", ident, exc_info=True) | |
949 | return |
|
953 | return | |
950 | self.log.info("registration::unregister_engine(%r)", eid) |
|
954 | self.log.info("registration::unregister_engine(%r)", eid) | |
951 | # print (eid) |
|
955 | # print (eid) | |
952 | uuid = self.keytable[eid] |
|
956 | uuid = self.keytable[eid] | |
953 | content=dict(id=eid, queue=uuid.decode('ascii')) |
|
957 | content=dict(id=eid, queue=uuid.decode('ascii')) | |
954 | self.dead_engines.add(uuid) |
|
958 | self.dead_engines.add(uuid) | |
955 | # self.ids.remove(eid) |
|
959 | # self.ids.remove(eid) | |
956 | # uuid = self.keytable.pop(eid) |
|
960 | # uuid = self.keytable.pop(eid) | |
957 | # |
|
961 | # | |
958 | # ec = self.engines.pop(eid) |
|
962 | # ec = self.engines.pop(eid) | |
959 | # self.hearts.pop(ec.heartbeat) |
|
963 | # self.hearts.pop(ec.heartbeat) | |
960 | # self.by_ident.pop(ec.queue) |
|
964 | # self.by_ident.pop(ec.queue) | |
961 | # self.completed.pop(eid) |
|
965 | # self.completed.pop(eid) | |
962 | handleit = lambda : self._handle_stranded_msgs(eid, uuid) |
|
966 | handleit = lambda : self._handle_stranded_msgs(eid, uuid) | |
963 | dc = ioloop.DelayedCallback(handleit, self.registration_timeout, self.loop) |
|
967 | dc = ioloop.DelayedCallback(handleit, self.registration_timeout, self.loop) | |
964 | dc.start() |
|
968 | dc.start() | |
965 | ############## TODO: HANDLE IT ################ |
|
969 | ############## TODO: HANDLE IT ################ | |
966 |
|
970 | |||
967 | if self.notifier: |
|
971 | if self.notifier: | |
968 | self.session.send(self.notifier, "unregistration_notification", content=content) |
|
972 | self.session.send(self.notifier, "unregistration_notification", content=content) | |
969 |
|
973 | |||
970 | def _handle_stranded_msgs(self, eid, uuid): |
|
974 | def _handle_stranded_msgs(self, eid, uuid): | |
971 | """Handle messages known to be on an engine when the engine unregisters. |
|
975 | """Handle messages known to be on an engine when the engine unregisters. | |
972 |
|
976 | |||
973 | It is possible that this will fire prematurely - that is, an engine will |
|
977 | It is possible that this will fire prematurely - that is, an engine will | |
974 | go down after completing a result, and the client will be notified |
|
978 | go down after completing a result, and the client will be notified | |
975 | that the result failed and later receive the actual result. |
|
979 | that the result failed and later receive the actual result. | |
976 | """ |
|
980 | """ | |
977 |
|
981 | |||
978 | outstanding = self.queues[eid] |
|
982 | outstanding = self.queues[eid] | |
979 |
|
983 | |||
980 | for msg_id in outstanding: |
|
984 | for msg_id in outstanding: | |
981 | self.pending.remove(msg_id) |
|
985 | self.pending.remove(msg_id) | |
982 | self.all_completed.add(msg_id) |
|
986 | self.all_completed.add(msg_id) | |
983 | try: |
|
987 | try: | |
984 | raise error.EngineError("Engine %r died while running task %r" % (eid, msg_id)) |
|
988 | raise error.EngineError("Engine %r died while running task %r" % (eid, msg_id)) | |
985 | except: |
|
989 | except: | |
986 | content = error.wrap_exception() |
|
990 | content = error.wrap_exception() | |
987 | # build a fake header: |
|
991 | # build a fake header: | |
988 | header = {} |
|
992 | header = {} | |
989 | header['engine'] = uuid |
|
993 | header['engine'] = uuid | |
990 | header['date'] = datetime.now() |
|
994 | header['date'] = datetime.now() | |
991 | rec = dict(result_content=content, result_header=header, result_buffers=[]) |
|
995 | rec = dict(result_content=content, result_header=header, result_buffers=[]) | |
992 | rec['completed'] = header['date'] |
|
996 | rec['completed'] = header['date'] | |
993 | rec['engine_uuid'] = uuid |
|
997 | rec['engine_uuid'] = uuid | |
994 | try: |
|
998 | try: | |
995 | self.db.update_record(msg_id, rec) |
|
999 | self.db.update_record(msg_id, rec) | |
996 | except Exception: |
|
1000 | except Exception: | |
997 | self.log.error("DB Error handling stranded msg %r", msg_id, exc_info=True) |
|
1001 | self.log.error("DB Error handling stranded msg %r", msg_id, exc_info=True) | |
998 |
|
1002 | |||
999 |
|
1003 | |||
1000 | def finish_registration(self, heart): |
|
1004 | def finish_registration(self, heart): | |
1001 | """Second half of engine registration, called after our HeartMonitor |
|
1005 | """Second half of engine registration, called after our HeartMonitor | |
1002 | has received a beat from the Engine's Heart.""" |
|
1006 | has received a beat from the Engine's Heart.""" | |
1003 | try: |
|
1007 | try: | |
1004 | (eid,queue,reg,purge) = self.incoming_registrations.pop(heart) |
|
1008 | (eid,queue,reg,purge) = self.incoming_registrations.pop(heart) | |
1005 | except KeyError: |
|
1009 | except KeyError: | |
1006 | self.log.error("registration::tried to finish nonexistant registration", exc_info=True) |
|
1010 | self.log.error("registration::tried to finish nonexistant registration", exc_info=True) | |
1007 | return |
|
1011 | return | |
1008 | self.log.info("registration::finished registering engine %i:%r", eid, queue) |
|
1012 | self.log.info("registration::finished registering engine %i:%r", eid, queue) | |
1009 | if purge is not None: |
|
1013 | if purge is not None: | |
1010 | purge.stop() |
|
1014 | purge.stop() | |
1011 | control = queue |
|
1015 | control = queue | |
1012 | self.ids.add(eid) |
|
1016 | self.ids.add(eid) | |
1013 | self.keytable[eid] = queue |
|
1017 | self.keytable[eid] = queue | |
1014 | self.engines[eid] = EngineConnector(id=eid, queue=queue, registration=reg, |
|
1018 | self.engines[eid] = EngineConnector(id=eid, queue=queue, registration=reg, | |
1015 | control=control, heartbeat=heart) |
|
1019 | control=control, heartbeat=heart) | |
1016 | self.by_ident[queue] = eid |
|
1020 | self.by_ident[queue] = eid | |
1017 | self.queues[eid] = list() |
|
1021 | self.queues[eid] = list() | |
1018 | self.tasks[eid] = list() |
|
1022 | self.tasks[eid] = list() | |
1019 | self.completed[eid] = list() |
|
1023 | self.completed[eid] = list() | |
1020 | self.hearts[heart] = eid |
|
1024 | self.hearts[heart] = eid | |
1021 | content = dict(id=eid, queue=self.engines[eid].queue.decode('ascii')) |
|
1025 | content = dict(id=eid, queue=self.engines[eid].queue.decode('ascii')) | |
1022 | if self.notifier: |
|
1026 | if self.notifier: | |
1023 | self.session.send(self.notifier, "registration_notification", content=content) |
|
1027 | self.session.send(self.notifier, "registration_notification", content=content) | |
1024 | self.log.info("engine::Engine Connected: %i", eid) |
|
1028 | self.log.info("engine::Engine Connected: %i", eid) | |
1025 |
|
1029 | |||
1026 | def _purge_stalled_registration(self, heart): |
|
1030 | def _purge_stalled_registration(self, heart): | |
1027 | if heart in self.incoming_registrations: |
|
1031 | if heart in self.incoming_registrations: | |
1028 | eid = self.incoming_registrations.pop(heart)[0] |
|
1032 | eid = self.incoming_registrations.pop(heart)[0] | |
1029 | self.log.info("registration::purging stalled registration: %i", eid) |
|
1033 | self.log.info("registration::purging stalled registration: %i", eid) | |
1030 | else: |
|
1034 | else: | |
1031 | pass |
|
1035 | pass | |
1032 |
|
1036 | |||
1033 | #------------------------------------------------------------------------- |
|
1037 | #------------------------------------------------------------------------- | |
1034 | # Client Requests |
|
1038 | # Client Requests | |
1035 | #------------------------------------------------------------------------- |
|
1039 | #------------------------------------------------------------------------- | |
1036 |
|
1040 | |||
1037 | def shutdown_request(self, client_id, msg): |
|
1041 | def shutdown_request(self, client_id, msg): | |
1038 | """handle shutdown request.""" |
|
1042 | """handle shutdown request.""" | |
1039 | self.session.send(self.query, 'shutdown_reply', content={'status': 'ok'}, ident=client_id) |
|
1043 | self.session.send(self.query, 'shutdown_reply', content={'status': 'ok'}, ident=client_id) | |
1040 | # also notify other clients of shutdown |
|
1044 | # also notify other clients of shutdown | |
1041 | self.session.send(self.notifier, 'shutdown_notice', content={'status': 'ok'}) |
|
1045 | self.session.send(self.notifier, 'shutdown_notice', content={'status': 'ok'}) | |
1042 | dc = ioloop.DelayedCallback(lambda : self._shutdown(), 1000, self.loop) |
|
1046 | dc = ioloop.DelayedCallback(lambda : self._shutdown(), 1000, self.loop) | |
1043 | dc.start() |
|
1047 | dc.start() | |
1044 |
|
1048 | |||
1045 | def _shutdown(self): |
|
1049 | def _shutdown(self): | |
1046 | self.log.info("hub::hub shutting down.") |
|
1050 | self.log.info("hub::hub shutting down.") | |
1047 | time.sleep(0.1) |
|
1051 | time.sleep(0.1) | |
1048 | sys.exit(0) |
|
1052 | sys.exit(0) | |
1049 |
|
1053 | |||
1050 |
|
1054 | |||
1051 | def check_load(self, client_id, msg): |
|
1055 | def check_load(self, client_id, msg): | |
1052 | content = msg['content'] |
|
1056 | content = msg['content'] | |
1053 | try: |
|
1057 | try: | |
1054 | targets = content['targets'] |
|
1058 | targets = content['targets'] | |
1055 | targets = self._validate_targets(targets) |
|
1059 | targets = self._validate_targets(targets) | |
1056 | except: |
|
1060 | except: | |
1057 | content = error.wrap_exception() |
|
1061 | content = error.wrap_exception() | |
1058 | self.session.send(self.query, "hub_error", |
|
1062 | self.session.send(self.query, "hub_error", | |
1059 | content=content, ident=client_id) |
|
1063 | content=content, ident=client_id) | |
1060 | return |
|
1064 | return | |
1061 |
|
1065 | |||
1062 | content = dict(status='ok') |
|
1066 | content = dict(status='ok') | |
1063 | # loads = {} |
|
1067 | # loads = {} | |
1064 | for t in targets: |
|
1068 | for t in targets: | |
1065 | content[bytes(t)] = len(self.queues[t])+len(self.tasks[t]) |
|
1069 | content[bytes(t)] = len(self.queues[t])+len(self.tasks[t]) | |
1066 | self.session.send(self.query, "load_reply", content=content, ident=client_id) |
|
1070 | self.session.send(self.query, "load_reply", content=content, ident=client_id) | |
1067 |
|
1071 | |||
1068 |
|
1072 | |||
1069 | def queue_status(self, client_id, msg): |
|
1073 | def queue_status(self, client_id, msg): | |
1070 | """Return the Queue status of one or more targets. |
|
1074 | """Return the Queue status of one or more targets. | |
1071 | if verbose: return the msg_ids |
|
1075 | if verbose: return the msg_ids | |
1072 | else: return len of each type. |
|
1076 | else: return len of each type. | |
1073 | keys: queue (pending MUX jobs) |
|
1077 | keys: queue (pending MUX jobs) | |
1074 | tasks (pending Task jobs) |
|
1078 | tasks (pending Task jobs) | |
1075 | completed (finished jobs from both queues)""" |
|
1079 | completed (finished jobs from both queues)""" | |
1076 | content = msg['content'] |
|
1080 | content = msg['content'] | |
1077 | targets = content['targets'] |
|
1081 | targets = content['targets'] | |
1078 | try: |
|
1082 | try: | |
1079 | targets = self._validate_targets(targets) |
|
1083 | targets = self._validate_targets(targets) | |
1080 | except: |
|
1084 | except: | |
1081 | content = error.wrap_exception() |
|
1085 | content = error.wrap_exception() | |
1082 | self.session.send(self.query, "hub_error", |
|
1086 | self.session.send(self.query, "hub_error", | |
1083 | content=content, ident=client_id) |
|
1087 | content=content, ident=client_id) | |
1084 | return |
|
1088 | return | |
1085 | verbose = content.get('verbose', False) |
|
1089 | verbose = content.get('verbose', False) | |
1086 | content = dict(status='ok') |
|
1090 | content = dict(status='ok') | |
1087 | for t in targets: |
|
1091 | for t in targets: | |
1088 | queue = self.queues[t] |
|
1092 | queue = self.queues[t] | |
1089 | completed = self.completed[t] |
|
1093 | completed = self.completed[t] | |
1090 | tasks = self.tasks[t] |
|
1094 | tasks = self.tasks[t] | |
1091 | if not verbose: |
|
1095 | if not verbose: | |
1092 | queue = len(queue) |
|
1096 | queue = len(queue) | |
1093 | completed = len(completed) |
|
1097 | completed = len(completed) | |
1094 | tasks = len(tasks) |
|
1098 | tasks = len(tasks) | |
1095 | content[str(t)] = {'queue': queue, 'completed': completed , 'tasks': tasks} |
|
1099 | content[str(t)] = {'queue': queue, 'completed': completed , 'tasks': tasks} | |
1096 | content['unassigned'] = list(self.unassigned) if verbose else len(self.unassigned) |
|
1100 | content['unassigned'] = list(self.unassigned) if verbose else len(self.unassigned) | |
1097 | # print (content) |
|
1101 | # print (content) | |
1098 | self.session.send(self.query, "queue_reply", content=content, ident=client_id) |
|
1102 | self.session.send(self.query, "queue_reply", content=content, ident=client_id) | |
1099 |
|
1103 | |||
1100 | def purge_results(self, client_id, msg): |
|
1104 | def purge_results(self, client_id, msg): | |
1101 | """Purge results from memory. This method is more valuable before we move |
|
1105 | """Purge results from memory. This method is more valuable before we move | |
1102 | to a DB based message storage mechanism.""" |
|
1106 | to a DB based message storage mechanism.""" | |
1103 | content = msg['content'] |
|
1107 | content = msg['content'] | |
1104 | self.log.info("Dropping records with %s", content) |
|
1108 | self.log.info("Dropping records with %s", content) | |
1105 | msg_ids = content.get('msg_ids', []) |
|
1109 | msg_ids = content.get('msg_ids', []) | |
1106 | reply = dict(status='ok') |
|
1110 | reply = dict(status='ok') | |
1107 | if msg_ids == 'all': |
|
1111 | if msg_ids == 'all': | |
1108 | try: |
|
1112 | try: | |
1109 | self.db.drop_matching_records(dict(completed={'$ne':None})) |
|
1113 | self.db.drop_matching_records(dict(completed={'$ne':None})) | |
1110 | except Exception: |
|
1114 | except Exception: | |
1111 | reply = error.wrap_exception() |
|
1115 | reply = error.wrap_exception() | |
1112 | else: |
|
1116 | else: | |
1113 | pending = filter(lambda m: m in self.pending, msg_ids) |
|
1117 | pending = filter(lambda m: m in self.pending, msg_ids) | |
1114 | if pending: |
|
1118 | if pending: | |
1115 | try: |
|
1119 | try: | |
1116 | raise IndexError("msg pending: %r" % pending[0]) |
|
1120 | raise IndexError("msg pending: %r" % pending[0]) | |
1117 | except: |
|
1121 | except: | |
1118 | reply = error.wrap_exception() |
|
1122 | reply = error.wrap_exception() | |
1119 | else: |
|
1123 | else: | |
1120 | try: |
|
1124 | try: | |
1121 | self.db.drop_matching_records(dict(msg_id={'$in':msg_ids})) |
|
1125 | self.db.drop_matching_records(dict(msg_id={'$in':msg_ids})) | |
1122 | except Exception: |
|
1126 | except Exception: | |
1123 | reply = error.wrap_exception() |
|
1127 | reply = error.wrap_exception() | |
1124 |
|
1128 | |||
1125 | if reply['status'] == 'ok': |
|
1129 | if reply['status'] == 'ok': | |
1126 | eids = content.get('engine_ids', []) |
|
1130 | eids = content.get('engine_ids', []) | |
1127 | for eid in eids: |
|
1131 | for eid in eids: | |
1128 | if eid not in self.engines: |
|
1132 | if eid not in self.engines: | |
1129 | try: |
|
1133 | try: | |
1130 | raise IndexError("No such engine: %i" % eid) |
|
1134 | raise IndexError("No such engine: %i" % eid) | |
1131 | except: |
|
1135 | except: | |
1132 | reply = error.wrap_exception() |
|
1136 | reply = error.wrap_exception() | |
1133 | break |
|
1137 | break | |
1134 | uid = self.engines[eid].queue |
|
1138 | uid = self.engines[eid].queue | |
1135 | try: |
|
1139 | try: | |
1136 | self.db.drop_matching_records(dict(engine_uuid=uid, completed={'$ne':None})) |
|
1140 | self.db.drop_matching_records(dict(engine_uuid=uid, completed={'$ne':None})) | |
1137 | except Exception: |
|
1141 | except Exception: | |
1138 | reply = error.wrap_exception() |
|
1142 | reply = error.wrap_exception() | |
1139 | break |
|
1143 | break | |
1140 |
|
1144 | |||
1141 | self.session.send(self.query, 'purge_reply', content=reply, ident=client_id) |
|
1145 | self.session.send(self.query, 'purge_reply', content=reply, ident=client_id) | |
1142 |
|
1146 | |||
1143 | def resubmit_task(self, client_id, msg): |
|
1147 | def resubmit_task(self, client_id, msg): | |
1144 | """Resubmit one or more tasks.""" |
|
1148 | """Resubmit one or more tasks.""" | |
1145 | def finish(reply): |
|
1149 | def finish(reply): | |
1146 | self.session.send(self.query, 'resubmit_reply', content=reply, ident=client_id) |
|
1150 | self.session.send(self.query, 'resubmit_reply', content=reply, ident=client_id) | |
1147 |
|
1151 | |||
1148 | content = msg['content'] |
|
1152 | content = msg['content'] | |
1149 | msg_ids = content['msg_ids'] |
|
1153 | msg_ids = content['msg_ids'] | |
1150 | reply = dict(status='ok') |
|
1154 | reply = dict(status='ok') | |
1151 | try: |
|
1155 | try: | |
1152 | records = self.db.find_records({'msg_id' : {'$in' : msg_ids}}, keys=[ |
|
1156 | records = self.db.find_records({'msg_id' : {'$in' : msg_ids}}, keys=[ | |
1153 | 'header', 'content', 'buffers']) |
|
1157 | 'header', 'content', 'buffers']) | |
1154 | except Exception: |
|
1158 | except Exception: | |
1155 | self.log.error('db::db error finding tasks to resubmit', exc_info=True) |
|
1159 | self.log.error('db::db error finding tasks to resubmit', exc_info=True) | |
1156 | return finish(error.wrap_exception()) |
|
1160 | return finish(error.wrap_exception()) | |
1157 |
|
1161 | |||
1158 | # validate msg_ids |
|
1162 | # validate msg_ids | |
1159 | found_ids = [ rec['msg_id'] for rec in records ] |
|
1163 | found_ids = [ rec['msg_id'] for rec in records ] | |
1160 | pending_ids = [ msg_id for msg_id in found_ids if msg_id in self.pending ] |
|
1164 | pending_ids = [ msg_id for msg_id in found_ids if msg_id in self.pending ] | |
1161 | if len(records) > len(msg_ids): |
|
1165 | if len(records) > len(msg_ids): | |
1162 | try: |
|
1166 | try: | |
1163 | raise RuntimeError("DB appears to be in an inconsistent state." |
|
1167 | raise RuntimeError("DB appears to be in an inconsistent state." | |
1164 | "More matching records were found than should exist") |
|
1168 | "More matching records were found than should exist") | |
1165 | except Exception: |
|
1169 | except Exception: | |
1166 | return finish(error.wrap_exception()) |
|
1170 | return finish(error.wrap_exception()) | |
1167 | elif len(records) < len(msg_ids): |
|
1171 | elif len(records) < len(msg_ids): | |
1168 | missing = [ m for m in msg_ids if m not in found_ids ] |
|
1172 | missing = [ m for m in msg_ids if m not in found_ids ] | |
1169 | try: |
|
1173 | try: | |
1170 | raise KeyError("No such msg(s): %r" % missing) |
|
1174 | raise KeyError("No such msg(s): %r" % missing) | |
1171 | except KeyError: |
|
1175 | except KeyError: | |
1172 | return finish(error.wrap_exception()) |
|
1176 | return finish(error.wrap_exception()) | |
1173 | elif pending_ids: |
|
1177 | elif pending_ids: | |
1174 | pass |
|
1178 | pass | |
1175 | # no need to raise on resubmit of pending task, now that we |
|
1179 | # no need to raise on resubmit of pending task, now that we | |
1176 | # resubmit under new ID, but do we want to raise anyway? |
|
1180 | # resubmit under new ID, but do we want to raise anyway? | |
1177 | # msg_id = invalid_ids[0] |
|
1181 | # msg_id = invalid_ids[0] | |
1178 | # try: |
|
1182 | # try: | |
1179 | # raise ValueError("Task(s) %r appears to be inflight" % ) |
|
1183 | # raise ValueError("Task(s) %r appears to be inflight" % ) | |
1180 | # except Exception: |
|
1184 | # except Exception: | |
1181 | # return finish(error.wrap_exception()) |
|
1185 | # return finish(error.wrap_exception()) | |
1182 |
|
1186 | |||
1183 | # mapping of original IDs to resubmitted IDs |
|
1187 | # mapping of original IDs to resubmitted IDs | |
1184 | resubmitted = {} |
|
1188 | resubmitted = {} | |
1185 |
|
1189 | |||
1186 | # send the messages |
|
1190 | # send the messages | |
1187 | for rec in records: |
|
1191 | for rec in records: | |
1188 | header = rec['header'] |
|
1192 | header = rec['header'] | |
1189 | msg = self.session.msg(header['msg_type'], parent=header) |
|
1193 | msg = self.session.msg(header['msg_type'], parent=header) | |
1190 | msg_id = msg['msg_id'] |
|
1194 | msg_id = msg['msg_id'] | |
1191 | msg['content'] = rec['content'] |
|
1195 | msg['content'] = rec['content'] | |
1192 |
|
1196 | |||
1193 | # use the old header, but update msg_id and timestamp |
|
1197 | # use the old header, but update msg_id and timestamp | |
1194 | fresh = msg['header'] |
|
1198 | fresh = msg['header'] | |
1195 | header['msg_id'] = fresh['msg_id'] |
|
1199 | header['msg_id'] = fresh['msg_id'] | |
1196 | header['date'] = fresh['date'] |
|
1200 | header['date'] = fresh['date'] | |
1197 | msg['header'] = header |
|
1201 | msg['header'] = header | |
1198 |
|
1202 | |||
1199 | self.session.send(self.resubmit, msg, buffers=rec['buffers']) |
|
1203 | self.session.send(self.resubmit, msg, buffers=rec['buffers']) | |
1200 |
|
1204 | |||
1201 | resubmitted[rec['msg_id']] = msg_id |
|
1205 | resubmitted[rec['msg_id']] = msg_id | |
1202 | self.pending.add(msg_id) |
|
1206 | self.pending.add(msg_id) | |
1203 | msg['buffers'] = rec['buffers'] |
|
1207 | msg['buffers'] = rec['buffers'] | |
1204 | try: |
|
1208 | try: | |
1205 | self.db.add_record(msg_id, init_record(msg)) |
|
1209 | self.db.add_record(msg_id, init_record(msg)) | |
1206 | except Exception: |
|
1210 | except Exception: | |
1207 | self.log.error("db::DB Error updating record: %s", msg_id, exc_info=True) |
|
1211 | self.log.error("db::DB Error updating record: %s", msg_id, exc_info=True) | |
1208 | return finish(error.wrap_exception()) |
|
1212 | return finish(error.wrap_exception()) | |
1209 |
|
1213 | |||
1210 | finish(dict(status='ok', resubmitted=resubmitted)) |
|
1214 | finish(dict(status='ok', resubmitted=resubmitted)) | |
1211 |
|
1215 | |||
1212 | # store the new IDs in the Task DB |
|
1216 | # store the new IDs in the Task DB | |
1213 | for msg_id, resubmit_id in resubmitted.iteritems(): |
|
1217 | for msg_id, resubmit_id in resubmitted.iteritems(): | |
1214 | try: |
|
1218 | try: | |
1215 | self.db.update_record(msg_id, {'resubmitted' : resubmit_id}) |
|
1219 | self.db.update_record(msg_id, {'resubmitted' : resubmit_id}) | |
1216 | except Exception: |
|
1220 | except Exception: | |
1217 | self.log.error("db::DB Error updating record: %s", msg_id, exc_info=True) |
|
1221 | self.log.error("db::DB Error updating record: %s", msg_id, exc_info=True) | |
1218 |
|
1222 | |||
1219 |
|
1223 | |||
1220 | def _extract_record(self, rec): |
|
1224 | def _extract_record(self, rec): | |
1221 | """decompose a TaskRecord dict into subsection of reply for get_result""" |
|
1225 | """decompose a TaskRecord dict into subsection of reply for get_result""" | |
1222 | io_dict = {} |
|
1226 | io_dict = {} | |
1223 | for key in ('pyin', 'pyout', 'pyerr', 'stdout', 'stderr'): |
|
1227 | for key in ('pyin', 'pyout', 'pyerr', 'stdout', 'stderr'): | |
1224 | io_dict[key] = rec[key] |
|
1228 | io_dict[key] = rec[key] | |
1225 | content = { 'result_content': rec['result_content'], |
|
1229 | content = { 'result_content': rec['result_content'], | |
1226 | 'header': rec['header'], |
|
1230 | 'header': rec['header'], | |
1227 | 'result_header' : rec['result_header'], |
|
1231 | 'result_header' : rec['result_header'], | |
1228 | 'received' : rec['received'], |
|
1232 | 'received' : rec['received'], | |
1229 | 'io' : io_dict, |
|
1233 | 'io' : io_dict, | |
1230 | } |
|
1234 | } | |
1231 | if rec['result_buffers']: |
|
1235 | if rec['result_buffers']: | |
1232 | buffers = map(bytes, rec['result_buffers']) |
|
1236 | buffers = map(bytes, rec['result_buffers']) | |
1233 | else: |
|
1237 | else: | |
1234 | buffers = [] |
|
1238 | buffers = [] | |
1235 |
|
1239 | |||
1236 | return content, buffers |
|
1240 | return content, buffers | |
1237 |
|
1241 | |||
1238 | def get_results(self, client_id, msg): |
|
1242 | def get_results(self, client_id, msg): | |
1239 | """Get the result of 1 or more messages.""" |
|
1243 | """Get the result of 1 or more messages.""" | |
1240 | content = msg['content'] |
|
1244 | content = msg['content'] | |
1241 | msg_ids = sorted(set(content['msg_ids'])) |
|
1245 | msg_ids = sorted(set(content['msg_ids'])) | |
1242 | statusonly = content.get('status_only', False) |
|
1246 | statusonly = content.get('status_only', False) | |
1243 | pending = [] |
|
1247 | pending = [] | |
1244 | completed = [] |
|
1248 | completed = [] | |
1245 | content = dict(status='ok') |
|
1249 | content = dict(status='ok') | |
1246 | content['pending'] = pending |
|
1250 | content['pending'] = pending | |
1247 | content['completed'] = completed |
|
1251 | content['completed'] = completed | |
1248 | buffers = [] |
|
1252 | buffers = [] | |
1249 | if not statusonly: |
|
1253 | if not statusonly: | |
1250 | try: |
|
1254 | try: | |
1251 | matches = self.db.find_records(dict(msg_id={'$in':msg_ids})) |
|
1255 | matches = self.db.find_records(dict(msg_id={'$in':msg_ids})) | |
1252 | # turn match list into dict, for faster lookup |
|
1256 | # turn match list into dict, for faster lookup | |
1253 | records = {} |
|
1257 | records = {} | |
1254 | for rec in matches: |
|
1258 | for rec in matches: | |
1255 | records[rec['msg_id']] = rec |
|
1259 | records[rec['msg_id']] = rec | |
1256 | except Exception: |
|
1260 | except Exception: | |
1257 | content = error.wrap_exception() |
|
1261 | content = error.wrap_exception() | |
1258 | self.session.send(self.query, "result_reply", content=content, |
|
1262 | self.session.send(self.query, "result_reply", content=content, | |
1259 | parent=msg, ident=client_id) |
|
1263 | parent=msg, ident=client_id) | |
1260 | return |
|
1264 | return | |
1261 | else: |
|
1265 | else: | |
1262 | records = {} |
|
1266 | records = {} | |
1263 | for msg_id in msg_ids: |
|
1267 | for msg_id in msg_ids: | |
1264 | if msg_id in self.pending: |
|
1268 | if msg_id in self.pending: | |
1265 | pending.append(msg_id) |
|
1269 | pending.append(msg_id) | |
1266 | elif msg_id in self.all_completed: |
|
1270 | elif msg_id in self.all_completed: | |
1267 | completed.append(msg_id) |
|
1271 | completed.append(msg_id) | |
1268 | if not statusonly: |
|
1272 | if not statusonly: | |
1269 | c,bufs = self._extract_record(records[msg_id]) |
|
1273 | c,bufs = self._extract_record(records[msg_id]) | |
1270 | content[msg_id] = c |
|
1274 | content[msg_id] = c | |
1271 | buffers.extend(bufs) |
|
1275 | buffers.extend(bufs) | |
1272 | elif msg_id in records: |
|
1276 | elif msg_id in records: | |
1273 | if rec['completed']: |
|
1277 | if rec['completed']: | |
1274 | completed.append(msg_id) |
|
1278 | completed.append(msg_id) | |
1275 | c,bufs = self._extract_record(records[msg_id]) |
|
1279 | c,bufs = self._extract_record(records[msg_id]) | |
1276 | content[msg_id] = c |
|
1280 | content[msg_id] = c | |
1277 | buffers.extend(bufs) |
|
1281 | buffers.extend(bufs) | |
1278 | else: |
|
1282 | else: | |
1279 | pending.append(msg_id) |
|
1283 | pending.append(msg_id) | |
1280 | else: |
|
1284 | else: | |
1281 | try: |
|
1285 | try: | |
1282 | raise KeyError('No such message: '+msg_id) |
|
1286 | raise KeyError('No such message: '+msg_id) | |
1283 | except: |
|
1287 | except: | |
1284 | content = error.wrap_exception() |
|
1288 | content = error.wrap_exception() | |
1285 | break |
|
1289 | break | |
1286 | self.session.send(self.query, "result_reply", content=content, |
|
1290 | self.session.send(self.query, "result_reply", content=content, | |
1287 | parent=msg, ident=client_id, |
|
1291 | parent=msg, ident=client_id, | |
1288 | buffers=buffers) |
|
1292 | buffers=buffers) | |
1289 |
|
1293 | |||
1290 | def get_history(self, client_id, msg): |
|
1294 | def get_history(self, client_id, msg): | |
1291 | """Get a list of all msg_ids in our DB records""" |
|
1295 | """Get a list of all msg_ids in our DB records""" | |
1292 | try: |
|
1296 | try: | |
1293 | msg_ids = self.db.get_history() |
|
1297 | msg_ids = self.db.get_history() | |
1294 | except Exception as e: |
|
1298 | except Exception as e: | |
1295 | content = error.wrap_exception() |
|
1299 | content = error.wrap_exception() | |
1296 | else: |
|
1300 | else: | |
1297 | content = dict(status='ok', history=msg_ids) |
|
1301 | content = dict(status='ok', history=msg_ids) | |
1298 |
|
1302 | |||
1299 | self.session.send(self.query, "history_reply", content=content, |
|
1303 | self.session.send(self.query, "history_reply", content=content, | |
1300 | parent=msg, ident=client_id) |
|
1304 | parent=msg, ident=client_id) | |
1301 |
|
1305 | |||
1302 | def db_query(self, client_id, msg): |
|
1306 | def db_query(self, client_id, msg): | |
1303 | """Perform a raw query on the task record database.""" |
|
1307 | """Perform a raw query on the task record database.""" | |
1304 | content = msg['content'] |
|
1308 | content = msg['content'] | |
1305 | query = content.get('query', {}) |
|
1309 | query = content.get('query', {}) | |
1306 | keys = content.get('keys', None) |
|
1310 | keys = content.get('keys', None) | |
1307 | buffers = [] |
|
1311 | buffers = [] | |
1308 | empty = list() |
|
1312 | empty = list() | |
1309 | try: |
|
1313 | try: | |
1310 | records = self.db.find_records(query, keys) |
|
1314 | records = self.db.find_records(query, keys) | |
1311 | except Exception as e: |
|
1315 | except Exception as e: | |
1312 | content = error.wrap_exception() |
|
1316 | content = error.wrap_exception() | |
1313 | else: |
|
1317 | else: | |
1314 | # extract buffers from reply content: |
|
1318 | # extract buffers from reply content: | |
1315 | if keys is not None: |
|
1319 | if keys is not None: | |
1316 | buffer_lens = [] if 'buffers' in keys else None |
|
1320 | buffer_lens = [] if 'buffers' in keys else None | |
1317 | result_buffer_lens = [] if 'result_buffers' in keys else None |
|
1321 | result_buffer_lens = [] if 'result_buffers' in keys else None | |
1318 | else: |
|
1322 | else: | |
1319 | buffer_lens = None |
|
1323 | buffer_lens = None | |
1320 | result_buffer_lens = None |
|
1324 | result_buffer_lens = None | |
1321 |
|
1325 | |||
1322 | for rec in records: |
|
1326 | for rec in records: | |
1323 | # buffers may be None, so double check |
|
1327 | # buffers may be None, so double check | |
1324 | b = rec.pop('buffers', empty) or empty |
|
1328 | b = rec.pop('buffers', empty) or empty | |
1325 | if buffer_lens is not None: |
|
1329 | if buffer_lens is not None: | |
1326 | buffer_lens.append(len(b)) |
|
1330 | buffer_lens.append(len(b)) | |
1327 | buffers.extend(b) |
|
1331 | buffers.extend(b) | |
1328 | rb = rec.pop('result_buffers', empty) or empty |
|
1332 | rb = rec.pop('result_buffers', empty) or empty | |
1329 | if result_buffer_lens is not None: |
|
1333 | if result_buffer_lens is not None: | |
1330 | result_buffer_lens.append(len(rb)) |
|
1334 | result_buffer_lens.append(len(rb)) | |
1331 | buffers.extend(rb) |
|
1335 | buffers.extend(rb) | |
1332 | content = dict(status='ok', records=records, buffer_lens=buffer_lens, |
|
1336 | content = dict(status='ok', records=records, buffer_lens=buffer_lens, | |
1333 | result_buffer_lens=result_buffer_lens) |
|
1337 | result_buffer_lens=result_buffer_lens) | |
1334 | # self.log.debug (content) |
|
1338 | # self.log.debug (content) | |
1335 | self.session.send(self.query, "db_reply", content=content, |
|
1339 | self.session.send(self.query, "db_reply", content=content, | |
1336 | parent=msg, ident=client_id, |
|
1340 | parent=msg, ident=client_id, | |
1337 | buffers=buffers) |
|
1341 | buffers=buffers) | |
1338 |
|
1342 |
@@ -1,237 +1,227 b'' | |||||
1 | """A simple engine that talks to a controller over 0MQ. |
|
1 | """A simple engine that talks to a controller over 0MQ. | |
2 | it handles registration, etc. and launches a kernel |
|
2 | it handles registration, etc. and launches a kernel | |
3 | connected to the Controller's Schedulers. |
|
3 | connected to the Controller's Schedulers. | |
4 |
|
4 | |||
5 | Authors: |
|
5 | Authors: | |
6 |
|
6 | |||
7 | * Min RK |
|
7 | * Min RK | |
8 | """ |
|
8 | """ | |
9 | #----------------------------------------------------------------------------- |
|
9 | #----------------------------------------------------------------------------- | |
10 | # Copyright (C) 2010-2011 The IPython Development Team |
|
10 | # Copyright (C) 2010-2011 The IPython Development Team | |
11 | # |
|
11 | # | |
12 | # Distributed under the terms of the BSD License. The full license is in |
|
12 | # Distributed under the terms of the BSD License. The full license is in | |
13 | # the file COPYING, distributed as part of this software. |
|
13 | # the file COPYING, distributed as part of this software. | |
14 | #----------------------------------------------------------------------------- |
|
14 | #----------------------------------------------------------------------------- | |
15 |
|
15 | |||
16 | from __future__ import print_function |
|
16 | from __future__ import print_function | |
17 |
|
17 | |||
18 | import sys |
|
18 | import sys | |
19 | import time |
|
19 | import time | |
20 | from getpass import getpass |
|
20 | from getpass import getpass | |
21 |
|
21 | |||
22 | import zmq |
|
22 | import zmq | |
23 | from zmq.eventloop import ioloop, zmqstream |
|
23 | from zmq.eventloop import ioloop, zmqstream | |
24 |
|
24 | |||
25 | from IPython.external.ssh import tunnel |
|
25 | from IPython.external.ssh import tunnel | |
26 | # internal |
|
26 | # internal | |
27 | from IPython.utils.traitlets import ( |
|
27 | from IPython.utils.traitlets import ( | |
28 | Instance, Dict, Integer, Type, CFloat, Unicode, CBytes, Bool |
|
28 | Instance, Dict, Integer, Type, CFloat, Unicode, CBytes, Bool | |
29 | ) |
|
29 | ) | |
30 | from IPython.utils.py3compat import cast_bytes |
|
30 | from IPython.utils.py3compat import cast_bytes | |
31 |
|
31 | |||
32 | from IPython.parallel.controller.heartmonitor import Heart |
|
32 | from IPython.parallel.controller.heartmonitor import Heart | |
33 | from IPython.parallel.factory import RegistrationFactory |
|
33 | from IPython.parallel.factory import RegistrationFactory | |
34 | from IPython.parallel.util import disambiguate_url |
|
34 | from IPython.parallel.util import disambiguate_url | |
35 |
|
35 | |||
36 | from IPython.zmq.session import Message |
|
36 | from IPython.zmq.session import Message | |
37 | from IPython.zmq.ipkernel import Kernel |
|
37 | from IPython.zmq.ipkernel import Kernel | |
38 |
|
38 | |||
39 | class EngineFactory(RegistrationFactory): |
|
39 | class EngineFactory(RegistrationFactory): | |
40 | """IPython engine""" |
|
40 | """IPython engine""" | |
41 |
|
41 | |||
42 | # configurables: |
|
42 | # configurables: | |
43 | out_stream_factory=Type('IPython.zmq.iostream.OutStream', config=True, |
|
43 | out_stream_factory=Type('IPython.zmq.iostream.OutStream', config=True, | |
44 | help="""The OutStream for handling stdout/err. |
|
44 | help="""The OutStream for handling stdout/err. | |
45 | Typically 'IPython.zmq.iostream.OutStream'""") |
|
45 | Typically 'IPython.zmq.iostream.OutStream'""") | |
46 | display_hook_factory=Type('IPython.zmq.displayhook.ZMQDisplayHook', config=True, |
|
46 | display_hook_factory=Type('IPython.zmq.displayhook.ZMQDisplayHook', config=True, | |
47 | help="""The class for handling displayhook. |
|
47 | help="""The class for handling displayhook. | |
48 | Typically 'IPython.zmq.displayhook.ZMQDisplayHook'""") |
|
48 | Typically 'IPython.zmq.displayhook.ZMQDisplayHook'""") | |
49 | location=Unicode(config=True, |
|
49 | location=Unicode(config=True, | |
50 | help="""The location (an IP address) of the controller. This is |
|
50 | help="""The location (an IP address) of the controller. This is | |
51 | used for disambiguating URLs, to determine whether |
|
51 | used for disambiguating URLs, to determine whether | |
52 | loopback should be used to connect or the public address.""") |
|
52 | loopback should be used to connect or the public address.""") | |
53 |
timeout=CFloat( |
|
53 | timeout=CFloat(5, config=True, | |
54 | help="""The time (in seconds) to wait for the Controller to respond |
|
54 | help="""The time (in seconds) to wait for the Controller to respond | |
55 | to registration requests before giving up.""") |
|
55 | to registration requests before giving up.""") | |
56 | sshserver=Unicode(config=True, |
|
56 | sshserver=Unicode(config=True, | |
57 | help="""The SSH server to use for tunneling connections to the Controller.""") |
|
57 | help="""The SSH server to use for tunneling connections to the Controller.""") | |
58 | sshkey=Unicode(config=True, |
|
58 | sshkey=Unicode(config=True, | |
59 | help="""The SSH private key file to use when tunneling connections to the Controller.""") |
|
59 | help="""The SSH private key file to use when tunneling connections to the Controller.""") | |
60 | paramiko=Bool(sys.platform == 'win32', config=True, |
|
60 | paramiko=Bool(sys.platform == 'win32', config=True, | |
61 | help="""Whether to use paramiko instead of openssh for tunnels.""") |
|
61 | help="""Whether to use paramiko instead of openssh for tunnels.""") | |
62 |
|
62 | |||
63 | # not configurable: |
|
63 | # not configurable: | |
64 |
|
|
64 | connection_info = Dict() | |
65 | id=Integer(allow_none=True) |
|
65 | user_ns = Dict() | |
66 | registrar=Instance('zmq.eventloop.zmqstream.ZMQStream') |
|
66 | id = Integer(allow_none=True) | |
67 | kernel=Instance(Kernel) |
|
67 | registrar = Instance('zmq.eventloop.zmqstream.ZMQStream') | |
|
68 | kernel = Instance(Kernel) | |||
68 |
|
69 | |||
69 | bident = CBytes() |
|
70 | bident = CBytes() | |
70 | ident = Unicode() |
|
71 | ident = Unicode() | |
71 | def _ident_changed(self, name, old, new): |
|
72 | def _ident_changed(self, name, old, new): | |
72 | self.bident = cast_bytes(new) |
|
73 | self.bident = cast_bytes(new) | |
73 | using_ssh=Bool(False) |
|
74 | using_ssh=Bool(False) | |
74 |
|
75 | |||
75 |
|
76 | |||
76 | def __init__(self, **kwargs): |
|
77 | def __init__(self, **kwargs): | |
77 | super(EngineFactory, self).__init__(**kwargs) |
|
78 | super(EngineFactory, self).__init__(**kwargs) | |
78 | self.ident = self.session.session |
|
79 | self.ident = self.session.session | |
79 |
|
80 | |||
80 | def init_connector(self): |
|
81 | def init_connector(self): | |
81 | """construct connection function, which handles tunnels.""" |
|
82 | """construct connection function, which handles tunnels.""" | |
82 | self.using_ssh = bool(self.sshkey or self.sshserver) |
|
83 | self.using_ssh = bool(self.sshkey or self.sshserver) | |
83 |
|
84 | |||
84 | if self.sshkey and not self.sshserver: |
|
85 | if self.sshkey and not self.sshserver: | |
85 | # We are using ssh directly to the controller, tunneling localhost to localhost |
|
86 | # We are using ssh directly to the controller, tunneling localhost to localhost | |
86 | self.sshserver = self.url.split('://')[1].split(':')[0] |
|
87 | self.sshserver = self.url.split('://')[1].split(':')[0] | |
87 |
|
88 | |||
88 | if self.using_ssh: |
|
89 | if self.using_ssh: | |
89 | if tunnel.try_passwordless_ssh(self.sshserver, self.sshkey, self.paramiko): |
|
90 | if tunnel.try_passwordless_ssh(self.sshserver, self.sshkey, self.paramiko): | |
90 | password=False |
|
91 | password=False | |
91 | else: |
|
92 | else: | |
92 | password = getpass("SSH Password for %s: "%self.sshserver) |
|
93 | password = getpass("SSH Password for %s: "%self.sshserver) | |
93 | else: |
|
94 | else: | |
94 | password = False |
|
95 | password = False | |
95 |
|
96 | |||
96 | def connect(s, url): |
|
97 | def connect(s, url): | |
97 | url = disambiguate_url(url, self.location) |
|
98 | url = disambiguate_url(url, self.location) | |
98 | if self.using_ssh: |
|
99 | if self.using_ssh: | |
99 |
self.log.debug("Tunneling connection to %s via %s" |
|
100 | self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) | |
100 | return tunnel.tunnel_connection(s, url, self.sshserver, |
|
101 | return tunnel.tunnel_connection(s, url, self.sshserver, | |
101 | keyfile=self.sshkey, paramiko=self.paramiko, |
|
102 | keyfile=self.sshkey, paramiko=self.paramiko, | |
102 | password=password, |
|
103 | password=password, | |
103 | ) |
|
104 | ) | |
104 | else: |
|
105 | else: | |
105 | return s.connect(url) |
|
106 | return s.connect(url) | |
106 |
|
107 | |||
107 | def maybe_tunnel(url): |
|
108 | def maybe_tunnel(url): | |
108 | """like connect, but don't complete the connection (for use by heartbeat)""" |
|
109 | """like connect, but don't complete the connection (for use by heartbeat)""" | |
109 | url = disambiguate_url(url, self.location) |
|
110 | url = disambiguate_url(url, self.location) | |
110 | if self.using_ssh: |
|
111 | if self.using_ssh: | |
111 |
self.log.debug("Tunneling connection to %s via %s" |
|
112 | self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) | |
112 | url,tunnelobj = tunnel.open_tunnel(url, self.sshserver, |
|
113 | url,tunnelobj = tunnel.open_tunnel(url, self.sshserver, | |
113 | keyfile=self.sshkey, paramiko=self.paramiko, |
|
114 | keyfile=self.sshkey, paramiko=self.paramiko, | |
114 | password=password, |
|
115 | password=password, | |
115 | ) |
|
116 | ) | |
116 | return url |
|
117 | return str(url) | |
117 | return connect, maybe_tunnel |
|
118 | return connect, maybe_tunnel | |
118 |
|
119 | |||
119 | def register(self): |
|
120 | def register(self): | |
120 | """send the registration_request""" |
|
121 | """send the registration_request""" | |
121 |
|
122 | |||
122 | self.log.info("Registering with controller at %s"%self.url) |
|
123 | self.log.info("Registering with controller at %s"%self.url) | |
123 | ctx = self.context |
|
124 | ctx = self.context | |
124 | connect,maybe_tunnel = self.init_connector() |
|
125 | connect,maybe_tunnel = self.init_connector() | |
125 | reg = ctx.socket(zmq.DEALER) |
|
126 | reg = ctx.socket(zmq.DEALER) | |
126 | reg.setsockopt(zmq.IDENTITY, self.bident) |
|
127 | reg.setsockopt(zmq.IDENTITY, self.bident) | |
127 | connect(reg, self.url) |
|
128 | connect(reg, self.url) | |
128 | self.registrar = zmqstream.ZMQStream(reg, self.loop) |
|
129 | self.registrar = zmqstream.ZMQStream(reg, self.loop) | |
129 |
|
130 | |||
130 |
|
131 | |||
131 | content = dict(queue=self.ident, heartbeat=self.ident, control=self.ident) |
|
132 | content = dict(queue=self.ident, heartbeat=self.ident, control=self.ident) | |
132 | self.registrar.on_recv(lambda msg: self.complete_registration(msg, connect, maybe_tunnel)) |
|
133 | self.registrar.on_recv(lambda msg: self.complete_registration(msg, connect, maybe_tunnel)) | |
133 | # print (self.session.key) |
|
134 | # print (self.session.key) | |
134 | self.session.send(self.registrar, "registration_request",content=content) |
|
135 | self.session.send(self.registrar, "registration_request", content=content) | |
135 |
|
136 | |||
136 | def complete_registration(self, msg, connect, maybe_tunnel): |
|
137 | def complete_registration(self, msg, connect, maybe_tunnel): | |
137 | # print msg |
|
138 | # print msg | |
138 | self._abort_dc.stop() |
|
139 | self._abort_dc.stop() | |
139 | ctx = self.context |
|
140 | ctx = self.context | |
140 | loop = self.loop |
|
141 | loop = self.loop | |
141 | identity = self.bident |
|
142 | identity = self.bident | |
142 | idents,msg = self.session.feed_identities(msg) |
|
143 | idents,msg = self.session.feed_identities(msg) | |
143 |
msg = |
|
144 | msg = self.session.unserialize(msg) | |
144 |
|
145 | content = msg['content'] | ||
145 | if msg.content.status == 'ok': |
|
146 | info = self.connection_info | |
146 | self.id = int(msg.content.id) |
|
147 | ||
|
148 | if content['status'] == 'ok': | |||
|
149 | self.id = int(content['id']) | |||
147 |
|
150 | |||
148 | # launch heartbeat |
|
151 | # launch heartbeat | |
149 | hb_addrs = msg.content.heartbeat |
|
|||
150 |
|
||||
151 | # possibly forward hb ports with tunnels |
|
152 | # possibly forward hb ports with tunnels | |
152 |
hb_ |
|
153 | hb_ping = maybe_tunnel(info['hb_ping']) | |
153 | heart = Heart(*map(str, hb_addrs), heart_id=identity) |
|
154 | hb_pong = maybe_tunnel(info['hb_pong']) | |
|
155 | ||||
|
156 | heart = Heart(hb_ping, hb_pong, heart_id=identity) | |||
154 | heart.start() |
|
157 | heart.start() | |
155 |
|
158 | |||
156 |
# create Shell |
|
159 | # create Shell Connections (MUX, Task, etc.): | |
157 | queue_addr = msg.content.mux |
|
160 | shell_addrs = map(str, [info['mux'], info['task']]) | |
158 | shell_addrs = [ str(queue_addr) ] |
|
161 | ||
159 | task_addr = msg.content.task |
|
162 | # Use only one shell stream for mux and tasks | |
160 | if task_addr: |
|
|||
161 | shell_addrs.append(str(task_addr)) |
|
|||
162 |
|
||||
163 | # Uncomment this to go back to two-socket model |
|
|||
164 | # shell_streams = [] |
|
|||
165 | # for addr in shell_addrs: |
|
|||
166 | # stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) |
|
|||
167 | # stream.setsockopt(zmq.IDENTITY, identity) |
|
|||
168 | # stream.connect(disambiguate_url(addr, self.location)) |
|
|||
169 | # shell_streams.append(stream) |
|
|||
170 |
|
||||
171 | # Now use only one shell stream for mux and tasks |
|
|||
172 | stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) |
|
163 | stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) | |
173 | stream.setsockopt(zmq.IDENTITY, identity) |
|
164 | stream.setsockopt(zmq.IDENTITY, identity) | |
174 | shell_streams = [stream] |
|
165 | shell_streams = [stream] | |
175 | for addr in shell_addrs: |
|
166 | for addr in shell_addrs: | |
176 | connect(stream, addr) |
|
167 | connect(stream, addr) | |
177 | # end single stream-socket |
|
|||
178 |
|
168 | |||
179 | # control stream: |
|
169 | # control stream: | |
180 |
control_addr = str( |
|
170 | control_addr = str(info['control']) | |
181 | control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) |
|
171 | control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) | |
182 | control_stream.setsockopt(zmq.IDENTITY, identity) |
|
172 | control_stream.setsockopt(zmq.IDENTITY, identity) | |
183 | connect(control_stream, control_addr) |
|
173 | connect(control_stream, control_addr) | |
184 |
|
174 | |||
185 | # create iopub stream: |
|
175 | # create iopub stream: | |
186 |
iopub_addr = |
|
176 | iopub_addr = info['iopub'] | |
187 | iopub_socket = ctx.socket(zmq.PUB) |
|
177 | iopub_socket = ctx.socket(zmq.PUB) | |
188 | iopub_socket.setsockopt(zmq.IDENTITY, identity) |
|
178 | iopub_socket.setsockopt(zmq.IDENTITY, identity) | |
189 | connect(iopub_socket, iopub_addr) |
|
179 | connect(iopub_socket, iopub_addr) | |
190 |
|
180 | |||
191 | # disable history: |
|
181 | # disable history: | |
192 | self.config.HistoryManager.hist_file = ':memory:' |
|
182 | self.config.HistoryManager.hist_file = ':memory:' | |
193 |
|
183 | |||
194 | # Redirect input streams and set a display hook. |
|
184 | # Redirect input streams and set a display hook. | |
195 | if self.out_stream_factory: |
|
185 | if self.out_stream_factory: | |
196 | sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout') |
|
186 | sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout') | |
197 | sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id) |
|
187 | sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id) | |
198 | sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr') |
|
188 | sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr') | |
199 | sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id) |
|
189 | sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id) | |
200 | if self.display_hook_factory: |
|
190 | if self.display_hook_factory: | |
201 | sys.displayhook = self.display_hook_factory(self.session, iopub_socket) |
|
191 | sys.displayhook = self.display_hook_factory(self.session, iopub_socket) | |
202 | sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id) |
|
192 | sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id) | |
203 |
|
193 | |||
204 | self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, |
|
194 | self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, | |
205 | control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, |
|
195 | control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, | |
206 | loop=loop, user_ns=self.user_ns, log=self.log) |
|
196 | loop=loop, user_ns=self.user_ns, log=self.log) | |
207 | self.kernel.shell.display_pub.topic = cast_bytes('engine.%i.displaypub' % self.id) |
|
197 | self.kernel.shell.display_pub.topic = cast_bytes('engine.%i.displaypub' % self.id) | |
208 | self.kernel.start() |
|
198 | self.kernel.start() | |
209 |
|
199 | |||
210 |
|
200 | |||
211 | else: |
|
201 | else: | |
212 | self.log.fatal("Registration Failed: %s"%msg) |
|
202 | self.log.fatal("Registration Failed: %s"%msg) | |
213 | raise Exception("Registration Failed: %s"%msg) |
|
203 | raise Exception("Registration Failed: %s"%msg) | |
214 |
|
204 | |||
215 | self.log.info("Completed registration with id %i"%self.id) |
|
205 | self.log.info("Completed registration with id %i"%self.id) | |
216 |
|
206 | |||
217 |
|
207 | |||
218 | def abort(self): |
|
208 | def abort(self): | |
219 | self.log.fatal("Registration timed out after %.1f seconds"%self.timeout) |
|
209 | self.log.fatal("Registration timed out after %.1f seconds"%self.timeout) | |
220 | if self.url.startswith('127.'): |
|
210 | if self.url.startswith('127.'): | |
221 | self.log.fatal(""" |
|
211 | self.log.fatal(""" | |
222 | If the controller and engines are not on the same machine, |
|
212 | If the controller and engines are not on the same machine, | |
223 | you will have to instruct the controller to listen on an external IP (in ipcontroller_config.py): |
|
213 | you will have to instruct the controller to listen on an external IP (in ipcontroller_config.py): | |
224 | c.HubFactory.ip='*' # for all interfaces, internal and external |
|
214 | c.HubFactory.ip='*' # for all interfaces, internal and external | |
225 | c.HubFactory.ip='192.168.1.101' # or any interface that the engines can see |
|
215 | c.HubFactory.ip='192.168.1.101' # or any interface that the engines can see | |
226 | or tunnel connections via ssh. |
|
216 | or tunnel connections via ssh. | |
227 | """) |
|
217 | """) | |
228 | self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id)) |
|
218 | self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id)) | |
229 | time.sleep(1) |
|
219 | time.sleep(1) | |
230 | sys.exit(255) |
|
220 | sys.exit(255) | |
231 |
|
221 | |||
232 | def start(self): |
|
222 | def start(self): | |
233 | dc = ioloop.DelayedCallback(self.register, 0, self.loop) |
|
223 | dc = ioloop.DelayedCallback(self.register, 0, self.loop) | |
234 | dc.start() |
|
224 | dc.start() | |
235 | self._abort_dc = ioloop.DelayedCallback(self.abort, self.timeout*1000, self.loop) |
|
225 | self._abort_dc = ioloop.DelayedCallback(self.abort, self.timeout*1000, self.loop) | |
236 | self._abort_dc.start() |
|
226 | self._abort_dc.start() | |
237 |
|
227 |
@@ -1,756 +1,760 b'' | |||||
1 | """Session object for building, serializing, sending, and receiving messages in |
|
1 | """Session object for building, serializing, sending, and receiving messages in | |
2 | IPython. The Session object supports serialization, HMAC signatures, and |
|
2 | IPython. The Session object supports serialization, HMAC signatures, and | |
3 | metadata on messages. |
|
3 | metadata on messages. | |
4 |
|
4 | |||
5 | Also defined here are utilities for working with Sessions: |
|
5 | Also defined here are utilities for working with Sessions: | |
6 | * A SessionFactory to be used as a base class for configurables that work with |
|
6 | * A SessionFactory to be used as a base class for configurables that work with | |
7 | Sessions. |
|
7 | Sessions. | |
8 | * A Message object for convenience that allows attribute-access to the msg dict. |
|
8 | * A Message object for convenience that allows attribute-access to the msg dict. | |
9 |
|
9 | |||
10 | Authors: |
|
10 | Authors: | |
11 |
|
11 | |||
12 | * Min RK |
|
12 | * Min RK | |
13 | * Brian Granger |
|
13 | * Brian Granger | |
14 | * Fernando Perez |
|
14 | * Fernando Perez | |
15 | """ |
|
15 | """ | |
16 | #----------------------------------------------------------------------------- |
|
16 | #----------------------------------------------------------------------------- | |
17 | # Copyright (C) 2010-2011 The IPython Development Team |
|
17 | # Copyright (C) 2010-2011 The IPython Development Team | |
18 | # |
|
18 | # | |
19 | # Distributed under the terms of the BSD License. The full license is in |
|
19 | # Distributed under the terms of the BSD License. The full license is in | |
20 | # the file COPYING, distributed as part of this software. |
|
20 | # the file COPYING, distributed as part of this software. | |
21 | #----------------------------------------------------------------------------- |
|
21 | #----------------------------------------------------------------------------- | |
22 |
|
22 | |||
23 | #----------------------------------------------------------------------------- |
|
23 | #----------------------------------------------------------------------------- | |
24 | # Imports |
|
24 | # Imports | |
25 | #----------------------------------------------------------------------------- |
|
25 | #----------------------------------------------------------------------------- | |
26 |
|
26 | |||
27 | import hmac |
|
27 | import hmac | |
28 | import logging |
|
28 | import logging | |
29 | import os |
|
29 | import os | |
30 | import pprint |
|
30 | import pprint | |
31 | import uuid |
|
31 | import uuid | |
32 | from datetime import datetime |
|
32 | from datetime import datetime | |
33 |
|
33 | |||
34 | try: |
|
34 | try: | |
35 | import cPickle |
|
35 | import cPickle | |
36 | pickle = cPickle |
|
36 | pickle = cPickle | |
37 | except: |
|
37 | except: | |
38 | cPickle = None |
|
38 | cPickle = None | |
39 | import pickle |
|
39 | import pickle | |
40 |
|
40 | |||
41 | import zmq |
|
41 | import zmq | |
42 | from zmq.utils import jsonapi |
|
42 | from zmq.utils import jsonapi | |
43 | from zmq.eventloop.ioloop import IOLoop |
|
43 | from zmq.eventloop.ioloop import IOLoop | |
44 | from zmq.eventloop.zmqstream import ZMQStream |
|
44 | from zmq.eventloop.zmqstream import ZMQStream | |
45 |
|
45 | |||
46 | from IPython.config.application import Application, boolean_flag |
|
46 | from IPython.config.application import Application, boolean_flag | |
47 | from IPython.config.configurable import Configurable, LoggingConfigurable |
|
47 | from IPython.config.configurable import Configurable, LoggingConfigurable | |
48 | from IPython.utils.importstring import import_item |
|
48 | from IPython.utils.importstring import import_item | |
49 | from IPython.utils.jsonutil import extract_dates, squash_dates, date_default |
|
49 | from IPython.utils.jsonutil import extract_dates, squash_dates, date_default | |
50 | from IPython.utils.py3compat import str_to_bytes |
|
50 | from IPython.utils.py3compat import str_to_bytes | |
51 | from IPython.utils.traitlets import (CBytes, Unicode, Bool, Any, Instance, Set, |
|
51 | from IPython.utils.traitlets import (CBytes, Unicode, Bool, Any, Instance, Set, | |
52 | DottedObjectName, CUnicode) |
|
52 | DottedObjectName, CUnicode) | |
53 |
|
53 | |||
54 | #----------------------------------------------------------------------------- |
|
54 | #----------------------------------------------------------------------------- | |
55 | # utility functions |
|
55 | # utility functions | |
56 | #----------------------------------------------------------------------------- |
|
56 | #----------------------------------------------------------------------------- | |
57 |
|
57 | |||
58 | def squash_unicode(obj): |
|
58 | def squash_unicode(obj): | |
59 | """coerce unicode back to bytestrings.""" |
|
59 | """coerce unicode back to bytestrings.""" | |
60 | if isinstance(obj,dict): |
|
60 | if isinstance(obj,dict): | |
61 | for key in obj.keys(): |
|
61 | for key in obj.keys(): | |
62 | obj[key] = squash_unicode(obj[key]) |
|
62 | obj[key] = squash_unicode(obj[key]) | |
63 | if isinstance(key, unicode): |
|
63 | if isinstance(key, unicode): | |
64 | obj[squash_unicode(key)] = obj.pop(key) |
|
64 | obj[squash_unicode(key)] = obj.pop(key) | |
65 | elif isinstance(obj, list): |
|
65 | elif isinstance(obj, list): | |
66 | for i,v in enumerate(obj): |
|
66 | for i,v in enumerate(obj): | |
67 | obj[i] = squash_unicode(v) |
|
67 | obj[i] = squash_unicode(v) | |
68 | elif isinstance(obj, unicode): |
|
68 | elif isinstance(obj, unicode): | |
69 | obj = obj.encode('utf8') |
|
69 | obj = obj.encode('utf8') | |
70 | return obj |
|
70 | return obj | |
71 |
|
71 | |||
72 | #----------------------------------------------------------------------------- |
|
72 | #----------------------------------------------------------------------------- | |
73 | # globals and defaults |
|
73 | # globals and defaults | |
74 | #----------------------------------------------------------------------------- |
|
74 | #----------------------------------------------------------------------------- | |
75 |
|
75 | |||
76 |
|
76 | |||
77 | # ISO8601-ify datetime objects |
|
77 | # ISO8601-ify datetime objects | |
78 | json_packer = lambda obj: jsonapi.dumps(obj, default=date_default) |
|
78 | json_packer = lambda obj: jsonapi.dumps(obj, default=date_default) | |
79 | json_unpacker = lambda s: extract_dates(jsonapi.loads(s)) |
|
79 | json_unpacker = lambda s: extract_dates(jsonapi.loads(s)) | |
80 |
|
80 | |||
81 | pickle_packer = lambda o: pickle.dumps(o,-1) |
|
81 | pickle_packer = lambda o: pickle.dumps(o,-1) | |
82 | pickle_unpacker = pickle.loads |
|
82 | pickle_unpacker = pickle.loads | |
83 |
|
83 | |||
84 | default_packer = json_packer |
|
84 | default_packer = json_packer | |
85 | default_unpacker = json_unpacker |
|
85 | default_unpacker = json_unpacker | |
86 |
|
86 | |||
87 | DELIM=b"<IDS|MSG>" |
|
87 | DELIM=b"<IDS|MSG>" | |
88 |
|
88 | |||
89 |
|
89 | |||
90 | #----------------------------------------------------------------------------- |
|
90 | #----------------------------------------------------------------------------- | |
91 | # Mixin tools for apps that use Sessions |
|
91 | # Mixin tools for apps that use Sessions | |
92 | #----------------------------------------------------------------------------- |
|
92 | #----------------------------------------------------------------------------- | |
93 |
|
93 | |||
94 | session_aliases = dict( |
|
94 | session_aliases = dict( | |
95 | ident = 'Session.session', |
|
95 | ident = 'Session.session', | |
96 | user = 'Session.username', |
|
96 | user = 'Session.username', | |
97 | keyfile = 'Session.keyfile', |
|
97 | keyfile = 'Session.keyfile', | |
98 | ) |
|
98 | ) | |
99 |
|
99 | |||
100 | session_flags = { |
|
100 | session_flags = { | |
101 | 'secure' : ({'Session' : { 'key' : str_to_bytes(str(uuid.uuid4())), |
|
101 | 'secure' : ({'Session' : { 'key' : str_to_bytes(str(uuid.uuid4())), | |
102 | 'keyfile' : '' }}, |
|
102 | 'keyfile' : '' }}, | |
103 | """Use HMAC digests for authentication of messages. |
|
103 | """Use HMAC digests for authentication of messages. | |
104 | Setting this flag will generate a new UUID to use as the HMAC key. |
|
104 | Setting this flag will generate a new UUID to use as the HMAC key. | |
105 | """), |
|
105 | """), | |
106 | 'no-secure' : ({'Session' : { 'key' : b'', 'keyfile' : '' }}, |
|
106 | 'no-secure' : ({'Session' : { 'key' : b'', 'keyfile' : '' }}, | |
107 | """Don't authenticate messages."""), |
|
107 | """Don't authenticate messages."""), | |
108 | } |
|
108 | } | |
109 |
|
109 | |||
110 | def default_secure(cfg): |
|
110 | def default_secure(cfg): | |
111 | """Set the default behavior for a config environment to be secure. |
|
111 | """Set the default behavior for a config environment to be secure. | |
112 |
|
112 | |||
113 | If Session.key/keyfile have not been set, set Session.key to |
|
113 | If Session.key/keyfile have not been set, set Session.key to | |
114 | a new random UUID. |
|
114 | a new random UUID. | |
115 | """ |
|
115 | """ | |
116 |
|
116 | |||
117 | if 'Session' in cfg: |
|
117 | if 'Session' in cfg: | |
118 | if 'key' in cfg.Session or 'keyfile' in cfg.Session: |
|
118 | if 'key' in cfg.Session or 'keyfile' in cfg.Session: | |
119 | return |
|
119 | return | |
120 | # key/keyfile not specified, generate new UUID: |
|
120 | # key/keyfile not specified, generate new UUID: | |
121 | cfg.Session.key = str_to_bytes(str(uuid.uuid4())) |
|
121 | cfg.Session.key = str_to_bytes(str(uuid.uuid4())) | |
122 |
|
122 | |||
123 |
|
123 | |||
124 | #----------------------------------------------------------------------------- |
|
124 | #----------------------------------------------------------------------------- | |
125 | # Classes |
|
125 | # Classes | |
126 | #----------------------------------------------------------------------------- |
|
126 | #----------------------------------------------------------------------------- | |
127 |
|
127 | |||
128 | class SessionFactory(LoggingConfigurable): |
|
128 | class SessionFactory(LoggingConfigurable): | |
129 | """The Base class for configurables that have a Session, Context, logger, |
|
129 | """The Base class for configurables that have a Session, Context, logger, | |
130 | and IOLoop. |
|
130 | and IOLoop. | |
131 | """ |
|
131 | """ | |
132 |
|
132 | |||
133 | logname = Unicode('') |
|
133 | logname = Unicode('') | |
134 | def _logname_changed(self, name, old, new): |
|
134 | def _logname_changed(self, name, old, new): | |
135 | self.log = logging.getLogger(new) |
|
135 | self.log = logging.getLogger(new) | |
136 |
|
136 | |||
137 | # not configurable: |
|
137 | # not configurable: | |
138 | context = Instance('zmq.Context') |
|
138 | context = Instance('zmq.Context') | |
139 | def _context_default(self): |
|
139 | def _context_default(self): | |
140 | return zmq.Context.instance() |
|
140 | return zmq.Context.instance() | |
141 |
|
141 | |||
142 | session = Instance('IPython.zmq.session.Session') |
|
142 | session = Instance('IPython.zmq.session.Session') | |
143 |
|
143 | |||
144 | loop = Instance('zmq.eventloop.ioloop.IOLoop', allow_none=False) |
|
144 | loop = Instance('zmq.eventloop.ioloop.IOLoop', allow_none=False) | |
145 | def _loop_default(self): |
|
145 | def _loop_default(self): | |
146 | return IOLoop.instance() |
|
146 | return IOLoop.instance() | |
147 |
|
147 | |||
148 | def __init__(self, **kwargs): |
|
148 | def __init__(self, **kwargs): | |
149 | super(SessionFactory, self).__init__(**kwargs) |
|
149 | super(SessionFactory, self).__init__(**kwargs) | |
150 |
|
150 | |||
151 | if self.session is None: |
|
151 | if self.session is None: | |
152 | # construct the session |
|
152 | # construct the session | |
153 | self.session = Session(**kwargs) |
|
153 | self.session = Session(**kwargs) | |
154 |
|
154 | |||
155 |
|
155 | |||
156 | class Message(object): |
|
156 | class Message(object): | |
157 | """A simple message object that maps dict keys to attributes. |
|
157 | """A simple message object that maps dict keys to attributes. | |
158 |
|
158 | |||
159 | A Message can be created from a dict and a dict from a Message instance |
|
159 | A Message can be created from a dict and a dict from a Message instance | |
160 | simply by calling dict(msg_obj).""" |
|
160 | simply by calling dict(msg_obj).""" | |
161 |
|
161 | |||
162 | def __init__(self, msg_dict): |
|
162 | def __init__(self, msg_dict): | |
163 | dct = self.__dict__ |
|
163 | dct = self.__dict__ | |
164 | for k, v in dict(msg_dict).iteritems(): |
|
164 | for k, v in dict(msg_dict).iteritems(): | |
165 | if isinstance(v, dict): |
|
165 | if isinstance(v, dict): | |
166 | v = Message(v) |
|
166 | v = Message(v) | |
167 | dct[k] = v |
|
167 | dct[k] = v | |
168 |
|
168 | |||
169 | # Having this iterator lets dict(msg_obj) work out of the box. |
|
169 | # Having this iterator lets dict(msg_obj) work out of the box. | |
170 | def __iter__(self): |
|
170 | def __iter__(self): | |
171 | return iter(self.__dict__.iteritems()) |
|
171 | return iter(self.__dict__.iteritems()) | |
172 |
|
172 | |||
173 | def __repr__(self): |
|
173 | def __repr__(self): | |
174 | return repr(self.__dict__) |
|
174 | return repr(self.__dict__) | |
175 |
|
175 | |||
176 | def __str__(self): |
|
176 | def __str__(self): | |
177 | return pprint.pformat(self.__dict__) |
|
177 | return pprint.pformat(self.__dict__) | |
178 |
|
178 | |||
179 | def __contains__(self, k): |
|
179 | def __contains__(self, k): | |
180 | return k in self.__dict__ |
|
180 | return k in self.__dict__ | |
181 |
|
181 | |||
182 | def __getitem__(self, k): |
|
182 | def __getitem__(self, k): | |
183 | return self.__dict__[k] |
|
183 | return self.__dict__[k] | |
184 |
|
184 | |||
185 |
|
185 | |||
186 | def msg_header(msg_id, msg_type, username, session): |
|
186 | def msg_header(msg_id, msg_type, username, session): | |
187 | date = datetime.now() |
|
187 | date = datetime.now() | |
188 | return locals() |
|
188 | return locals() | |
189 |
|
189 | |||
190 | def extract_header(msg_or_header): |
|
190 | def extract_header(msg_or_header): | |
191 | """Given a message or header, return the header.""" |
|
191 | """Given a message or header, return the header.""" | |
192 | if not msg_or_header: |
|
192 | if not msg_or_header: | |
193 | return {} |
|
193 | return {} | |
194 | try: |
|
194 | try: | |
195 | # See if msg_or_header is the entire message. |
|
195 | # See if msg_or_header is the entire message. | |
196 | h = msg_or_header['header'] |
|
196 | h = msg_or_header['header'] | |
197 | except KeyError: |
|
197 | except KeyError: | |
198 | try: |
|
198 | try: | |
199 | # See if msg_or_header is just the header |
|
199 | # See if msg_or_header is just the header | |
200 | h = msg_or_header['msg_id'] |
|
200 | h = msg_or_header['msg_id'] | |
201 | except KeyError: |
|
201 | except KeyError: | |
202 | raise |
|
202 | raise | |
203 | else: |
|
203 | else: | |
204 | h = msg_or_header |
|
204 | h = msg_or_header | |
205 | if not isinstance(h, dict): |
|
205 | if not isinstance(h, dict): | |
206 | h = dict(h) |
|
206 | h = dict(h) | |
207 | return h |
|
207 | return h | |
208 |
|
208 | |||
209 | class Session(Configurable): |
|
209 | class Session(Configurable): | |
210 | """Object for handling serialization and sending of messages. |
|
210 | """Object for handling serialization and sending of messages. | |
211 |
|
211 | |||
212 | The Session object handles building messages and sending them |
|
212 | The Session object handles building messages and sending them | |
213 | with ZMQ sockets or ZMQStream objects. Objects can communicate with each |
|
213 | with ZMQ sockets or ZMQStream objects. Objects can communicate with each | |
214 | other over the network via Session objects, and only need to work with the |
|
214 | other over the network via Session objects, and only need to work with the | |
215 | dict-based IPython message spec. The Session will handle |
|
215 | dict-based IPython message spec. The Session will handle | |
216 | serialization/deserialization, security, and metadata. |
|
216 | serialization/deserialization, security, and metadata. | |
217 |
|
217 | |||
218 | Sessions support configurable serialiization via packer/unpacker traits, |
|
218 | Sessions support configurable serialiization via packer/unpacker traits, | |
219 | and signing with HMAC digests via the key/keyfile traits. |
|
219 | and signing with HMAC digests via the key/keyfile traits. | |
220 |
|
220 | |||
221 | Parameters |
|
221 | Parameters | |
222 | ---------- |
|
222 | ---------- | |
223 |
|
223 | |||
224 | debug : bool |
|
224 | debug : bool | |
225 | whether to trigger extra debugging statements |
|
225 | whether to trigger extra debugging statements | |
226 | packer/unpacker : str : 'json', 'pickle' or import_string |
|
226 | packer/unpacker : str : 'json', 'pickle' or import_string | |
227 | importstrings for methods to serialize message parts. If just |
|
227 | importstrings for methods to serialize message parts. If just | |
228 | 'json' or 'pickle', predefined JSON and pickle packers will be used. |
|
228 | 'json' or 'pickle', predefined JSON and pickle packers will be used. | |
229 | Otherwise, the entire importstring must be used. |
|
229 | Otherwise, the entire importstring must be used. | |
230 |
|
230 | |||
231 | The functions must accept at least valid JSON input, and output *bytes*. |
|
231 | The functions must accept at least valid JSON input, and output *bytes*. | |
232 |
|
232 | |||
233 | For example, to use msgpack: |
|
233 | For example, to use msgpack: | |
234 | packer = 'msgpack.packb', unpacker='msgpack.unpackb' |
|
234 | packer = 'msgpack.packb', unpacker='msgpack.unpackb' | |
235 | pack/unpack : callables |
|
235 | pack/unpack : callables | |
236 | You can also set the pack/unpack callables for serialization directly. |
|
236 | You can also set the pack/unpack callables for serialization directly. | |
237 | session : bytes |
|
237 | session : bytes | |
238 | the ID of this Session object. The default is to generate a new UUID. |
|
238 | the ID of this Session object. The default is to generate a new UUID. | |
239 | username : unicode |
|
239 | username : unicode | |
240 | username added to message headers. The default is to ask the OS. |
|
240 | username added to message headers. The default is to ask the OS. | |
241 | key : bytes |
|
241 | key : bytes | |
242 | The key used to initialize an HMAC signature. If unset, messages |
|
242 | The key used to initialize an HMAC signature. If unset, messages | |
243 | will not be signed or checked. |
|
243 | will not be signed or checked. | |
244 | keyfile : filepath |
|
244 | keyfile : filepath | |
245 | The file containing a key. If this is set, `key` will be initialized |
|
245 | The file containing a key. If this is set, `key` will be initialized | |
246 | to the contents of the file. |
|
246 | to the contents of the file. | |
247 |
|
247 | |||
248 | """ |
|
248 | """ | |
249 |
|
249 | |||
250 | debug=Bool(False, config=True, help="""Debug output in the Session""") |
|
250 | debug=Bool(False, config=True, help="""Debug output in the Session""") | |
251 |
|
251 | |||
252 | packer = DottedObjectName('json',config=True, |
|
252 | packer = DottedObjectName('json',config=True, | |
253 | help="""The name of the packer for serializing messages. |
|
253 | help="""The name of the packer for serializing messages. | |
254 | Should be one of 'json', 'pickle', or an import name |
|
254 | Should be one of 'json', 'pickle', or an import name | |
255 | for a custom callable serializer.""") |
|
255 | for a custom callable serializer.""") | |
256 | def _packer_changed(self, name, old, new): |
|
256 | def _packer_changed(self, name, old, new): | |
257 | if new.lower() == 'json': |
|
257 | if new.lower() == 'json': | |
258 | self.pack = json_packer |
|
258 | self.pack = json_packer | |
259 | self.unpack = json_unpacker |
|
259 | self.unpack = json_unpacker | |
|
260 | self.unpacker = new | |||
260 | elif new.lower() == 'pickle': |
|
261 | elif new.lower() == 'pickle': | |
261 | self.pack = pickle_packer |
|
262 | self.pack = pickle_packer | |
262 | self.unpack = pickle_unpacker |
|
263 | self.unpack = pickle_unpacker | |
|
264 | self.unpacker = new | |||
263 | else: |
|
265 | else: | |
264 | self.pack = import_item(str(new)) |
|
266 | self.pack = import_item(str(new)) | |
265 |
|
267 | |||
266 | unpacker = DottedObjectName('json', config=True, |
|
268 | unpacker = DottedObjectName('json', config=True, | |
267 | help="""The name of the unpacker for unserializing messages. |
|
269 | help="""The name of the unpacker for unserializing messages. | |
268 | Only used with custom functions for `packer`.""") |
|
270 | Only used with custom functions for `packer`.""") | |
269 | def _unpacker_changed(self, name, old, new): |
|
271 | def _unpacker_changed(self, name, old, new): | |
270 | if new.lower() == 'json': |
|
272 | if new.lower() == 'json': | |
271 | self.pack = json_packer |
|
273 | self.pack = json_packer | |
272 | self.unpack = json_unpacker |
|
274 | self.unpack = json_unpacker | |
|
275 | self.packer = new | |||
273 | elif new.lower() == 'pickle': |
|
276 | elif new.lower() == 'pickle': | |
274 | self.pack = pickle_packer |
|
277 | self.pack = pickle_packer | |
275 | self.unpack = pickle_unpacker |
|
278 | self.unpack = pickle_unpacker | |
|
279 | self.packer = new | |||
276 | else: |
|
280 | else: | |
277 | self.unpack = import_item(str(new)) |
|
281 | self.unpack = import_item(str(new)) | |
278 |
|
282 | |||
279 | session = CUnicode(u'', config=True, |
|
283 | session = CUnicode(u'', config=True, | |
280 | help="""The UUID identifying this session.""") |
|
284 | help="""The UUID identifying this session.""") | |
281 | def _session_default(self): |
|
285 | def _session_default(self): | |
282 | u = unicode(uuid.uuid4()) |
|
286 | u = unicode(uuid.uuid4()) | |
283 | self.bsession = u.encode('ascii') |
|
287 | self.bsession = u.encode('ascii') | |
284 | return u |
|
288 | return u | |
285 |
|
289 | |||
286 | def _session_changed(self, name, old, new): |
|
290 | def _session_changed(self, name, old, new): | |
287 | self.bsession = self.session.encode('ascii') |
|
291 | self.bsession = self.session.encode('ascii') | |
288 |
|
292 | |||
289 | # bsession is the session as bytes |
|
293 | # bsession is the session as bytes | |
290 | bsession = CBytes(b'') |
|
294 | bsession = CBytes(b'') | |
291 |
|
295 | |||
292 | username = Unicode(os.environ.get('USER',u'username'), config=True, |
|
296 | username = Unicode(os.environ.get('USER',u'username'), config=True, | |
293 | help="""Username for the Session. Default is your system username.""") |
|
297 | help="""Username for the Session. Default is your system username.""") | |
294 |
|
298 | |||
295 | # message signature related traits: |
|
299 | # message signature related traits: | |
296 |
|
300 | |||
297 | key = CBytes(b'', config=True, |
|
301 | key = CBytes(b'', config=True, | |
298 | help="""execution key, for extra authentication.""") |
|
302 | help="""execution key, for extra authentication.""") | |
299 | def _key_changed(self, name, old, new): |
|
303 | def _key_changed(self, name, old, new): | |
300 | if new: |
|
304 | if new: | |
301 | self.auth = hmac.HMAC(new) |
|
305 | self.auth = hmac.HMAC(new) | |
302 | else: |
|
306 | else: | |
303 | self.auth = None |
|
307 | self.auth = None | |
304 | auth = Instance(hmac.HMAC) |
|
308 | auth = Instance(hmac.HMAC) | |
305 | digest_history = Set() |
|
309 | digest_history = Set() | |
306 |
|
310 | |||
307 | keyfile = Unicode('', config=True, |
|
311 | keyfile = Unicode('', config=True, | |
308 | help="""path to file containing execution key.""") |
|
312 | help="""path to file containing execution key.""") | |
309 | def _keyfile_changed(self, name, old, new): |
|
313 | def _keyfile_changed(self, name, old, new): | |
310 | with open(new, 'rb') as f: |
|
314 | with open(new, 'rb') as f: | |
311 | self.key = f.read().strip() |
|
315 | self.key = f.read().strip() | |
312 |
|
316 | |||
313 | # serialization traits: |
|
317 | # serialization traits: | |
314 |
|
318 | |||
315 | pack = Any(default_packer) # the actual packer function |
|
319 | pack = Any(default_packer) # the actual packer function | |
316 | def _pack_changed(self, name, old, new): |
|
320 | def _pack_changed(self, name, old, new): | |
317 | if not callable(new): |
|
321 | if not callable(new): | |
318 | raise TypeError("packer must be callable, not %s"%type(new)) |
|
322 | raise TypeError("packer must be callable, not %s"%type(new)) | |
319 |
|
323 | |||
320 | unpack = Any(default_unpacker) # the actual packer function |
|
324 | unpack = Any(default_unpacker) # the actual packer function | |
321 | def _unpack_changed(self, name, old, new): |
|
325 | def _unpack_changed(self, name, old, new): | |
322 | # unpacker is not checked - it is assumed to be |
|
326 | # unpacker is not checked - it is assumed to be | |
323 | if not callable(new): |
|
327 | if not callable(new): | |
324 | raise TypeError("unpacker must be callable, not %s"%type(new)) |
|
328 | raise TypeError("unpacker must be callable, not %s"%type(new)) | |
325 |
|
329 | |||
326 | def __init__(self, **kwargs): |
|
330 | def __init__(self, **kwargs): | |
327 | """create a Session object |
|
331 | """create a Session object | |
328 |
|
332 | |||
329 | Parameters |
|
333 | Parameters | |
330 | ---------- |
|
334 | ---------- | |
331 |
|
335 | |||
332 | debug : bool |
|
336 | debug : bool | |
333 | whether to trigger extra debugging statements |
|
337 | whether to trigger extra debugging statements | |
334 | packer/unpacker : str : 'json', 'pickle' or import_string |
|
338 | packer/unpacker : str : 'json', 'pickle' or import_string | |
335 | importstrings for methods to serialize message parts. If just |
|
339 | importstrings for methods to serialize message parts. If just | |
336 | 'json' or 'pickle', predefined JSON and pickle packers will be used. |
|
340 | 'json' or 'pickle', predefined JSON and pickle packers will be used. | |
337 | Otherwise, the entire importstring must be used. |
|
341 | Otherwise, the entire importstring must be used. | |
338 |
|
342 | |||
339 | The functions must accept at least valid JSON input, and output |
|
343 | The functions must accept at least valid JSON input, and output | |
340 | *bytes*. |
|
344 | *bytes*. | |
341 |
|
345 | |||
342 | For example, to use msgpack: |
|
346 | For example, to use msgpack: | |
343 | packer = 'msgpack.packb', unpacker='msgpack.unpackb' |
|
347 | packer = 'msgpack.packb', unpacker='msgpack.unpackb' | |
344 | pack/unpack : callables |
|
348 | pack/unpack : callables | |
345 | You can also set the pack/unpack callables for serialization |
|
349 | You can also set the pack/unpack callables for serialization | |
346 | directly. |
|
350 | directly. | |
347 | session : unicode (must be ascii) |
|
351 | session : unicode (must be ascii) | |
348 | the ID of this Session object. The default is to generate a new |
|
352 | the ID of this Session object. The default is to generate a new | |
349 | UUID. |
|
353 | UUID. | |
350 | bsession : bytes |
|
354 | bsession : bytes | |
351 | The session as bytes |
|
355 | The session as bytes | |
352 | username : unicode |
|
356 | username : unicode | |
353 | username added to message headers. The default is to ask the OS. |
|
357 | username added to message headers. The default is to ask the OS. | |
354 | key : bytes |
|
358 | key : bytes | |
355 | The key used to initialize an HMAC signature. If unset, messages |
|
359 | The key used to initialize an HMAC signature. If unset, messages | |
356 | will not be signed or checked. |
|
360 | will not be signed or checked. | |
357 | keyfile : filepath |
|
361 | keyfile : filepath | |
358 | The file containing a key. If this is set, `key` will be |
|
362 | The file containing a key. If this is set, `key` will be | |
359 | initialized to the contents of the file. |
|
363 | initialized to the contents of the file. | |
360 | """ |
|
364 | """ | |
361 | super(Session, self).__init__(**kwargs) |
|
365 | super(Session, self).__init__(**kwargs) | |
362 | self._check_packers() |
|
366 | self._check_packers() | |
363 | self.none = self.pack({}) |
|
367 | self.none = self.pack({}) | |
364 | # ensure self._session_default() if necessary, so bsession is defined: |
|
368 | # ensure self._session_default() if necessary, so bsession is defined: | |
365 | self.session |
|
369 | self.session | |
366 |
|
370 | |||
367 | @property |
|
371 | @property | |
368 | def msg_id(self): |
|
372 | def msg_id(self): | |
369 | """always return new uuid""" |
|
373 | """always return new uuid""" | |
370 | return str(uuid.uuid4()) |
|
374 | return str(uuid.uuid4()) | |
371 |
|
375 | |||
372 | def _check_packers(self): |
|
376 | def _check_packers(self): | |
373 | """check packers for binary data and datetime support.""" |
|
377 | """check packers for binary data and datetime support.""" | |
374 | pack = self.pack |
|
378 | pack = self.pack | |
375 | unpack = self.unpack |
|
379 | unpack = self.unpack | |
376 |
|
380 | |||
377 | # check simple serialization |
|
381 | # check simple serialization | |
378 | msg = dict(a=[1,'hi']) |
|
382 | msg = dict(a=[1,'hi']) | |
379 | try: |
|
383 | try: | |
380 | packed = pack(msg) |
|
384 | packed = pack(msg) | |
381 | except Exception: |
|
385 | except Exception: | |
382 | raise ValueError("packer could not serialize a simple message") |
|
386 | raise ValueError("packer could not serialize a simple message") | |
383 |
|
387 | |||
384 | # ensure packed message is bytes |
|
388 | # ensure packed message is bytes | |
385 | if not isinstance(packed, bytes): |
|
389 | if not isinstance(packed, bytes): | |
386 | raise ValueError("message packed to %r, but bytes are required"%type(packed)) |
|
390 | raise ValueError("message packed to %r, but bytes are required"%type(packed)) | |
387 |
|
391 | |||
388 | # check that unpack is pack's inverse |
|
392 | # check that unpack is pack's inverse | |
389 | try: |
|
393 | try: | |
390 | unpacked = unpack(packed) |
|
394 | unpacked = unpack(packed) | |
391 | except Exception: |
|
395 | except Exception: | |
392 | raise ValueError("unpacker could not handle the packer's output") |
|
396 | raise ValueError("unpacker could not handle the packer's output") | |
393 |
|
397 | |||
394 | # check datetime support |
|
398 | # check datetime support | |
395 | msg = dict(t=datetime.now()) |
|
399 | msg = dict(t=datetime.now()) | |
396 | try: |
|
400 | try: | |
397 | unpacked = unpack(pack(msg)) |
|
401 | unpacked = unpack(pack(msg)) | |
398 | except Exception: |
|
402 | except Exception: | |
399 | self.pack = lambda o: pack(squash_dates(o)) |
|
403 | self.pack = lambda o: pack(squash_dates(o)) | |
400 | self.unpack = lambda s: extract_dates(unpack(s)) |
|
404 | self.unpack = lambda s: extract_dates(unpack(s)) | |
401 |
|
405 | |||
402 | def msg_header(self, msg_type): |
|
406 | def msg_header(self, msg_type): | |
403 | return msg_header(self.msg_id, msg_type, self.username, self.session) |
|
407 | return msg_header(self.msg_id, msg_type, self.username, self.session) | |
404 |
|
408 | |||
405 | def msg(self, msg_type, content=None, parent=None, subheader=None, header=None): |
|
409 | def msg(self, msg_type, content=None, parent=None, subheader=None, header=None): | |
406 | """Return the nested message dict. |
|
410 | """Return the nested message dict. | |
407 |
|
411 | |||
408 | This format is different from what is sent over the wire. The |
|
412 | This format is different from what is sent over the wire. The | |
409 | serialize/unserialize methods converts this nested message dict to the wire |
|
413 | serialize/unserialize methods converts this nested message dict to the wire | |
410 | format, which is a list of message parts. |
|
414 | format, which is a list of message parts. | |
411 | """ |
|
415 | """ | |
412 | msg = {} |
|
416 | msg = {} | |
413 | header = self.msg_header(msg_type) if header is None else header |
|
417 | header = self.msg_header(msg_type) if header is None else header | |
414 | msg['header'] = header |
|
418 | msg['header'] = header | |
415 | msg['msg_id'] = header['msg_id'] |
|
419 | msg['msg_id'] = header['msg_id'] | |
416 | msg['msg_type'] = header['msg_type'] |
|
420 | msg['msg_type'] = header['msg_type'] | |
417 | msg['parent_header'] = {} if parent is None else extract_header(parent) |
|
421 | msg['parent_header'] = {} if parent is None else extract_header(parent) | |
418 | msg['content'] = {} if content is None else content |
|
422 | msg['content'] = {} if content is None else content | |
419 | sub = {} if subheader is None else subheader |
|
423 | sub = {} if subheader is None else subheader | |
420 | msg['header'].update(sub) |
|
424 | msg['header'].update(sub) | |
421 | return msg |
|
425 | return msg | |
422 |
|
426 | |||
423 | def sign(self, msg_list): |
|
427 | def sign(self, msg_list): | |
424 | """Sign a message with HMAC digest. If no auth, return b''. |
|
428 | """Sign a message with HMAC digest. If no auth, return b''. | |
425 |
|
429 | |||
426 | Parameters |
|
430 | Parameters | |
427 | ---------- |
|
431 | ---------- | |
428 | msg_list : list |
|
432 | msg_list : list | |
429 | The [p_header,p_parent,p_content] part of the message list. |
|
433 | The [p_header,p_parent,p_content] part of the message list. | |
430 | """ |
|
434 | """ | |
431 | if self.auth is None: |
|
435 | if self.auth is None: | |
432 | return b'' |
|
436 | return b'' | |
433 | h = self.auth.copy() |
|
437 | h = self.auth.copy() | |
434 | for m in msg_list: |
|
438 | for m in msg_list: | |
435 | h.update(m) |
|
439 | h.update(m) | |
436 | return str_to_bytes(h.hexdigest()) |
|
440 | return str_to_bytes(h.hexdigest()) | |
437 |
|
441 | |||
438 | def serialize(self, msg, ident=None): |
|
442 | def serialize(self, msg, ident=None): | |
439 | """Serialize the message components to bytes. |
|
443 | """Serialize the message components to bytes. | |
440 |
|
444 | |||
441 | This is roughly the inverse of unserialize. The serialize/unserialize |
|
445 | This is roughly the inverse of unserialize. The serialize/unserialize | |
442 | methods work with full message lists, whereas pack/unpack work with |
|
446 | methods work with full message lists, whereas pack/unpack work with | |
443 | the individual message parts in the message list. |
|
447 | the individual message parts in the message list. | |
444 |
|
448 | |||
445 | Parameters |
|
449 | Parameters | |
446 | ---------- |
|
450 | ---------- | |
447 | msg : dict or Message |
|
451 | msg : dict or Message | |
448 | The nexted message dict as returned by the self.msg method. |
|
452 | The nexted message dict as returned by the self.msg method. | |
449 |
|
453 | |||
450 | Returns |
|
454 | Returns | |
451 | ------- |
|
455 | ------- | |
452 | msg_list : list |
|
456 | msg_list : list | |
453 | The list of bytes objects to be sent with the format: |
|
457 | The list of bytes objects to be sent with the format: | |
454 | [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content, |
|
458 | [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content, | |
455 | buffer1,buffer2,...]. In this list, the p_* entities are |
|
459 | buffer1,buffer2,...]. In this list, the p_* entities are | |
456 | the packed or serialized versions, so if JSON is used, these |
|
460 | the packed or serialized versions, so if JSON is used, these | |
457 | are utf8 encoded JSON strings. |
|
461 | are utf8 encoded JSON strings. | |
458 | """ |
|
462 | """ | |
459 | content = msg.get('content', {}) |
|
463 | content = msg.get('content', {}) | |
460 | if content is None: |
|
464 | if content is None: | |
461 | content = self.none |
|
465 | content = self.none | |
462 | elif isinstance(content, dict): |
|
466 | elif isinstance(content, dict): | |
463 | content = self.pack(content) |
|
467 | content = self.pack(content) | |
464 | elif isinstance(content, bytes): |
|
468 | elif isinstance(content, bytes): | |
465 | # content is already packed, as in a relayed message |
|
469 | # content is already packed, as in a relayed message | |
466 | pass |
|
470 | pass | |
467 | elif isinstance(content, unicode): |
|
471 | elif isinstance(content, unicode): | |
468 | # should be bytes, but JSON often spits out unicode |
|
472 | # should be bytes, but JSON often spits out unicode | |
469 | content = content.encode('utf8') |
|
473 | content = content.encode('utf8') | |
470 | else: |
|
474 | else: | |
471 | raise TypeError("Content incorrect type: %s"%type(content)) |
|
475 | raise TypeError("Content incorrect type: %s"%type(content)) | |
472 |
|
476 | |||
473 | real_message = [self.pack(msg['header']), |
|
477 | real_message = [self.pack(msg['header']), | |
474 | self.pack(msg['parent_header']), |
|
478 | self.pack(msg['parent_header']), | |
475 | content |
|
479 | content | |
476 | ] |
|
480 | ] | |
477 |
|
481 | |||
478 | to_send = [] |
|
482 | to_send = [] | |
479 |
|
483 | |||
480 | if isinstance(ident, list): |
|
484 | if isinstance(ident, list): | |
481 | # accept list of idents |
|
485 | # accept list of idents | |
482 | to_send.extend(ident) |
|
486 | to_send.extend(ident) | |
483 | elif ident is not None: |
|
487 | elif ident is not None: | |
484 | to_send.append(ident) |
|
488 | to_send.append(ident) | |
485 | to_send.append(DELIM) |
|
489 | to_send.append(DELIM) | |
486 |
|
490 | |||
487 | signature = self.sign(real_message) |
|
491 | signature = self.sign(real_message) | |
488 | to_send.append(signature) |
|
492 | to_send.append(signature) | |
489 |
|
493 | |||
490 | to_send.extend(real_message) |
|
494 | to_send.extend(real_message) | |
491 |
|
495 | |||
492 | return to_send |
|
496 | return to_send | |
493 |
|
497 | |||
494 | def send(self, stream, msg_or_type, content=None, parent=None, ident=None, |
|
498 | def send(self, stream, msg_or_type, content=None, parent=None, ident=None, | |
495 | buffers=None, subheader=None, track=False, header=None): |
|
499 | buffers=None, subheader=None, track=False, header=None): | |
496 | """Build and send a message via stream or socket. |
|
500 | """Build and send a message via stream or socket. | |
497 |
|
501 | |||
498 | The message format used by this function internally is as follows: |
|
502 | The message format used by this function internally is as follows: | |
499 |
|
503 | |||
500 | [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content, |
|
504 | [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content, | |
501 | buffer1,buffer2,...] |
|
505 | buffer1,buffer2,...] | |
502 |
|
506 | |||
503 | The serialize/unserialize methods convert the nested message dict into this |
|
507 | The serialize/unserialize methods convert the nested message dict into this | |
504 | format. |
|
508 | format. | |
505 |
|
509 | |||
506 | Parameters |
|
510 | Parameters | |
507 | ---------- |
|
511 | ---------- | |
508 |
|
512 | |||
509 | stream : zmq.Socket or ZMQStream |
|
513 | stream : zmq.Socket or ZMQStream | |
510 | The socket-like object used to send the data. |
|
514 | The socket-like object used to send the data. | |
511 | msg_or_type : str or Message/dict |
|
515 | msg_or_type : str or Message/dict | |
512 | Normally, msg_or_type will be a msg_type unless a message is being |
|
516 | Normally, msg_or_type will be a msg_type unless a message is being | |
513 | sent more than once. If a header is supplied, this can be set to |
|
517 | sent more than once. If a header is supplied, this can be set to | |
514 | None and the msg_type will be pulled from the header. |
|
518 | None and the msg_type will be pulled from the header. | |
515 |
|
519 | |||
516 | content : dict or None |
|
520 | content : dict or None | |
517 | The content of the message (ignored if msg_or_type is a message). |
|
521 | The content of the message (ignored if msg_or_type is a message). | |
518 | header : dict or None |
|
522 | header : dict or None | |
519 | The header dict for the message (ignores if msg_to_type is a message). |
|
523 | The header dict for the message (ignores if msg_to_type is a message). | |
520 | parent : Message or dict or None |
|
524 | parent : Message or dict or None | |
521 | The parent or parent header describing the parent of this message |
|
525 | The parent or parent header describing the parent of this message | |
522 | (ignored if msg_or_type is a message). |
|
526 | (ignored if msg_or_type is a message). | |
523 | ident : bytes or list of bytes |
|
527 | ident : bytes or list of bytes | |
524 | The zmq.IDENTITY routing path. |
|
528 | The zmq.IDENTITY routing path. | |
525 | subheader : dict or None |
|
529 | subheader : dict or None | |
526 | Extra header keys for this message's header (ignored if msg_or_type |
|
530 | Extra header keys for this message's header (ignored if msg_or_type | |
527 | is a message). |
|
531 | is a message). | |
528 | buffers : list or None |
|
532 | buffers : list or None | |
529 | The already-serialized buffers to be appended to the message. |
|
533 | The already-serialized buffers to be appended to the message. | |
530 | track : bool |
|
534 | track : bool | |
531 | Whether to track. Only for use with Sockets, because ZMQStream |
|
535 | Whether to track. Only for use with Sockets, because ZMQStream | |
532 | objects cannot track messages. |
|
536 | objects cannot track messages. | |
533 |
|
537 | |||
534 | Returns |
|
538 | Returns | |
535 | ------- |
|
539 | ------- | |
536 | msg : dict |
|
540 | msg : dict | |
537 | The constructed message. |
|
541 | The constructed message. | |
538 | (msg,tracker) : (dict, MessageTracker) |
|
542 | (msg,tracker) : (dict, MessageTracker) | |
539 | if track=True, then a 2-tuple will be returned, |
|
543 | if track=True, then a 2-tuple will be returned, | |
540 | the first element being the constructed |
|
544 | the first element being the constructed | |
541 | message, and the second being the MessageTracker |
|
545 | message, and the second being the MessageTracker | |
542 |
|
546 | |||
543 | """ |
|
547 | """ | |
544 |
|
548 | |||
545 | if not isinstance(stream, (zmq.Socket, ZMQStream)): |
|
549 | if not isinstance(stream, (zmq.Socket, ZMQStream)): | |
546 | raise TypeError("stream must be Socket or ZMQStream, not %r"%type(stream)) |
|
550 | raise TypeError("stream must be Socket or ZMQStream, not %r"%type(stream)) | |
547 | elif track and isinstance(stream, ZMQStream): |
|
551 | elif track and isinstance(stream, ZMQStream): | |
548 | raise TypeError("ZMQStream cannot track messages") |
|
552 | raise TypeError("ZMQStream cannot track messages") | |
549 |
|
553 | |||
550 | if isinstance(msg_or_type, (Message, dict)): |
|
554 | if isinstance(msg_or_type, (Message, dict)): | |
551 | # We got a Message or message dict, not a msg_type so don't |
|
555 | # We got a Message or message dict, not a msg_type so don't | |
552 | # build a new Message. |
|
556 | # build a new Message. | |
553 | msg = msg_or_type |
|
557 | msg = msg_or_type | |
554 | else: |
|
558 | else: | |
555 | msg = self.msg(msg_or_type, content=content, parent=parent, |
|
559 | msg = self.msg(msg_or_type, content=content, parent=parent, | |
556 | subheader=subheader, header=header) |
|
560 | subheader=subheader, header=header) | |
557 |
|
561 | |||
558 | buffers = [] if buffers is None else buffers |
|
562 | buffers = [] if buffers is None else buffers | |
559 | to_send = self.serialize(msg, ident) |
|
563 | to_send = self.serialize(msg, ident) | |
560 | flag = 0 |
|
564 | flag = 0 | |
561 | if buffers: |
|
565 | if buffers: | |
562 | flag = zmq.SNDMORE |
|
566 | flag = zmq.SNDMORE | |
563 | _track = False |
|
567 | _track = False | |
564 | else: |
|
568 | else: | |
565 | _track=track |
|
569 | _track=track | |
566 | if track: |
|
570 | if track: | |
567 | tracker = stream.send_multipart(to_send, flag, copy=False, track=_track) |
|
571 | tracker = stream.send_multipart(to_send, flag, copy=False, track=_track) | |
568 | else: |
|
572 | else: | |
569 | tracker = stream.send_multipart(to_send, flag, copy=False) |
|
573 | tracker = stream.send_multipart(to_send, flag, copy=False) | |
570 | for b in buffers[:-1]: |
|
574 | for b in buffers[:-1]: | |
571 | stream.send(b, flag, copy=False) |
|
575 | stream.send(b, flag, copy=False) | |
572 | if buffers: |
|
576 | if buffers: | |
573 | if track: |
|
577 | if track: | |
574 | tracker = stream.send(buffers[-1], copy=False, track=track) |
|
578 | tracker = stream.send(buffers[-1], copy=False, track=track) | |
575 | else: |
|
579 | else: | |
576 | tracker = stream.send(buffers[-1], copy=False) |
|
580 | tracker = stream.send(buffers[-1], copy=False) | |
577 |
|
581 | |||
578 | # omsg = Message(msg) |
|
582 | # omsg = Message(msg) | |
579 | if self.debug: |
|
583 | if self.debug: | |
580 | pprint.pprint(msg) |
|
584 | pprint.pprint(msg) | |
581 | pprint.pprint(to_send) |
|
585 | pprint.pprint(to_send) | |
582 | pprint.pprint(buffers) |
|
586 | pprint.pprint(buffers) | |
583 |
|
587 | |||
584 | msg['tracker'] = tracker |
|
588 | msg['tracker'] = tracker | |
585 |
|
589 | |||
586 | return msg |
|
590 | return msg | |
587 |
|
591 | |||
588 | def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None): |
|
592 | def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None): | |
589 | """Send a raw message via ident path. |
|
593 | """Send a raw message via ident path. | |
590 |
|
594 | |||
591 | This method is used to send a already serialized message. |
|
595 | This method is used to send a already serialized message. | |
592 |
|
596 | |||
593 | Parameters |
|
597 | Parameters | |
594 | ---------- |
|
598 | ---------- | |
595 | stream : ZMQStream or Socket |
|
599 | stream : ZMQStream or Socket | |
596 | The ZMQ stream or socket to use for sending the message. |
|
600 | The ZMQ stream or socket to use for sending the message. | |
597 | msg_list : list |
|
601 | msg_list : list | |
598 | The serialized list of messages to send. This only includes the |
|
602 | The serialized list of messages to send. This only includes the | |
599 | [p_header,p_parent,p_content,buffer1,buffer2,...] portion of |
|
603 | [p_header,p_parent,p_content,buffer1,buffer2,...] portion of | |
600 | the message. |
|
604 | the message. | |
601 | ident : ident or list |
|
605 | ident : ident or list | |
602 | A single ident or a list of idents to use in sending. |
|
606 | A single ident or a list of idents to use in sending. | |
603 | """ |
|
607 | """ | |
604 | to_send = [] |
|
608 | to_send = [] | |
605 | if isinstance(ident, bytes): |
|
609 | if isinstance(ident, bytes): | |
606 | ident = [ident] |
|
610 | ident = [ident] | |
607 | if ident is not None: |
|
611 | if ident is not None: | |
608 | to_send.extend(ident) |
|
612 | to_send.extend(ident) | |
609 |
|
613 | |||
610 | to_send.append(DELIM) |
|
614 | to_send.append(DELIM) | |
611 | to_send.append(self.sign(msg_list)) |
|
615 | to_send.append(self.sign(msg_list)) | |
612 | to_send.extend(msg_list) |
|
616 | to_send.extend(msg_list) | |
613 | stream.send_multipart(msg_list, flags, copy=copy) |
|
617 | stream.send_multipart(msg_list, flags, copy=copy) | |
614 |
|
618 | |||
615 | def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True): |
|
619 | def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True): | |
616 | """Receive and unpack a message. |
|
620 | """Receive and unpack a message. | |
617 |
|
621 | |||
618 | Parameters |
|
622 | Parameters | |
619 | ---------- |
|
623 | ---------- | |
620 | socket : ZMQStream or Socket |
|
624 | socket : ZMQStream or Socket | |
621 | The socket or stream to use in receiving. |
|
625 | The socket or stream to use in receiving. | |
622 |
|
626 | |||
623 | Returns |
|
627 | Returns | |
624 | ------- |
|
628 | ------- | |
625 | [idents], msg |
|
629 | [idents], msg | |
626 | [idents] is a list of idents and msg is a nested message dict of |
|
630 | [idents] is a list of idents and msg is a nested message dict of | |
627 | same format as self.msg returns. |
|
631 | same format as self.msg returns. | |
628 | """ |
|
632 | """ | |
629 | if isinstance(socket, ZMQStream): |
|
633 | if isinstance(socket, ZMQStream): | |
630 | socket = socket.socket |
|
634 | socket = socket.socket | |
631 | try: |
|
635 | try: | |
632 | msg_list = socket.recv_multipart(mode, copy=copy) |
|
636 | msg_list = socket.recv_multipart(mode, copy=copy) | |
633 | except zmq.ZMQError as e: |
|
637 | except zmq.ZMQError as e: | |
634 | if e.errno == zmq.EAGAIN: |
|
638 | if e.errno == zmq.EAGAIN: | |
635 | # We can convert EAGAIN to None as we know in this case |
|
639 | # We can convert EAGAIN to None as we know in this case | |
636 | # recv_multipart won't return None. |
|
640 | # recv_multipart won't return None. | |
637 | return None,None |
|
641 | return None,None | |
638 | else: |
|
642 | else: | |
639 | raise |
|
643 | raise | |
640 | # split multipart message into identity list and message dict |
|
644 | # split multipart message into identity list and message dict | |
641 | # invalid large messages can cause very expensive string comparisons |
|
645 | # invalid large messages can cause very expensive string comparisons | |
642 | idents, msg_list = self.feed_identities(msg_list, copy) |
|
646 | idents, msg_list = self.feed_identities(msg_list, copy) | |
643 | try: |
|
647 | try: | |
644 | return idents, self.unserialize(msg_list, content=content, copy=copy) |
|
648 | return idents, self.unserialize(msg_list, content=content, copy=copy) | |
645 | except Exception as e: |
|
649 | except Exception as e: | |
646 | # TODO: handle it |
|
650 | # TODO: handle it | |
647 | raise e |
|
651 | raise e | |
648 |
|
652 | |||
649 | def feed_identities(self, msg_list, copy=True): |
|
653 | def feed_identities(self, msg_list, copy=True): | |
650 | """Split the identities from the rest of the message. |
|
654 | """Split the identities from the rest of the message. | |
651 |
|
655 | |||
652 | Feed until DELIM is reached, then return the prefix as idents and |
|
656 | Feed until DELIM is reached, then return the prefix as idents and | |
653 | remainder as msg_list. This is easily broken by setting an IDENT to DELIM, |
|
657 | remainder as msg_list. This is easily broken by setting an IDENT to DELIM, | |
654 | but that would be silly. |
|
658 | but that would be silly. | |
655 |
|
659 | |||
656 | Parameters |
|
660 | Parameters | |
657 | ---------- |
|
661 | ---------- | |
658 | msg_list : a list of Message or bytes objects |
|
662 | msg_list : a list of Message or bytes objects | |
659 | The message to be split. |
|
663 | The message to be split. | |
660 | copy : bool |
|
664 | copy : bool | |
661 | flag determining whether the arguments are bytes or Messages |
|
665 | flag determining whether the arguments are bytes or Messages | |
662 |
|
666 | |||
663 | Returns |
|
667 | Returns | |
664 | ------- |
|
668 | ------- | |
665 | (idents, msg_list) : two lists |
|
669 | (idents, msg_list) : two lists | |
666 | idents will always be a list of bytes, each of which is a ZMQ |
|
670 | idents will always be a list of bytes, each of which is a ZMQ | |
667 | identity. msg_list will be a list of bytes or zmq.Messages of the |
|
671 | identity. msg_list will be a list of bytes or zmq.Messages of the | |
668 | form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and |
|
672 | form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and | |
669 | should be unpackable/unserializable via self.unserialize at this |
|
673 | should be unpackable/unserializable via self.unserialize at this | |
670 | point. |
|
674 | point. | |
671 | """ |
|
675 | """ | |
672 | if copy: |
|
676 | if copy: | |
673 | idx = msg_list.index(DELIM) |
|
677 | idx = msg_list.index(DELIM) | |
674 | return msg_list[:idx], msg_list[idx+1:] |
|
678 | return msg_list[:idx], msg_list[idx+1:] | |
675 | else: |
|
679 | else: | |
676 | failed = True |
|
680 | failed = True | |
677 | for idx,m in enumerate(msg_list): |
|
681 | for idx,m in enumerate(msg_list): | |
678 | if m.bytes == DELIM: |
|
682 | if m.bytes == DELIM: | |
679 | failed = False |
|
683 | failed = False | |
680 | break |
|
684 | break | |
681 | if failed: |
|
685 | if failed: | |
682 | raise ValueError("DELIM not in msg_list") |
|
686 | raise ValueError("DELIM not in msg_list") | |
683 | idents, msg_list = msg_list[:idx], msg_list[idx+1:] |
|
687 | idents, msg_list = msg_list[:idx], msg_list[idx+1:] | |
684 | return [m.bytes for m in idents], msg_list |
|
688 | return [m.bytes for m in idents], msg_list | |
685 |
|
689 | |||
686 | def unserialize(self, msg_list, content=True, copy=True): |
|
690 | def unserialize(self, msg_list, content=True, copy=True): | |
687 | """Unserialize a msg_list to a nested message dict. |
|
691 | """Unserialize a msg_list to a nested message dict. | |
688 |
|
692 | |||
689 | This is roughly the inverse of serialize. The serialize/unserialize |
|
693 | This is roughly the inverse of serialize. The serialize/unserialize | |
690 | methods work with full message lists, whereas pack/unpack work with |
|
694 | methods work with full message lists, whereas pack/unpack work with | |
691 | the individual message parts in the message list. |
|
695 | the individual message parts in the message list. | |
692 |
|
696 | |||
693 | Parameters: |
|
697 | Parameters: | |
694 | ----------- |
|
698 | ----------- | |
695 | msg_list : list of bytes or Message objects |
|
699 | msg_list : list of bytes or Message objects | |
696 | The list of message parts of the form [HMAC,p_header,p_parent, |
|
700 | The list of message parts of the form [HMAC,p_header,p_parent, | |
697 | p_content,buffer1,buffer2,...]. |
|
701 | p_content,buffer1,buffer2,...]. | |
698 | content : bool (True) |
|
702 | content : bool (True) | |
699 | Whether to unpack the content dict (True), or leave it packed |
|
703 | Whether to unpack the content dict (True), or leave it packed | |
700 | (False). |
|
704 | (False). | |
701 | copy : bool (True) |
|
705 | copy : bool (True) | |
702 | Whether to return the bytes (True), or the non-copying Message |
|
706 | Whether to return the bytes (True), or the non-copying Message | |
703 | object in each place (False). |
|
707 | object in each place (False). | |
704 |
|
708 | |||
705 | Returns |
|
709 | Returns | |
706 | ------- |
|
710 | ------- | |
707 | msg : dict |
|
711 | msg : dict | |
708 | The nested message dict with top-level keys [header, parent_header, |
|
712 | The nested message dict with top-level keys [header, parent_header, | |
709 | content, buffers]. |
|
713 | content, buffers]. | |
710 | """ |
|
714 | """ | |
711 | minlen = 4 |
|
715 | minlen = 4 | |
712 | message = {} |
|
716 | message = {} | |
713 | if not copy: |
|
717 | if not copy: | |
714 | for i in range(minlen): |
|
718 | for i in range(minlen): | |
715 | msg_list[i] = msg_list[i].bytes |
|
719 | msg_list[i] = msg_list[i].bytes | |
716 | if self.auth is not None: |
|
720 | if self.auth is not None: | |
717 | signature = msg_list[0] |
|
721 | signature = msg_list[0] | |
718 | if not signature: |
|
722 | if not signature: | |
719 | raise ValueError("Unsigned Message") |
|
723 | raise ValueError("Unsigned Message") | |
720 | if signature in self.digest_history: |
|
724 | if signature in self.digest_history: | |
721 | raise ValueError("Duplicate Signature: %r"%signature) |
|
725 | raise ValueError("Duplicate Signature: %r"%signature) | |
722 | self.digest_history.add(signature) |
|
726 | self.digest_history.add(signature) | |
723 | check = self.sign(msg_list[1:4]) |
|
727 | check = self.sign(msg_list[1:4]) | |
724 | if not signature == check: |
|
728 | if not signature == check: | |
725 | raise ValueError("Invalid Signature: %r"%signature) |
|
729 | raise ValueError("Invalid Signature: %r"%signature) | |
726 | if not len(msg_list) >= minlen: |
|
730 | if not len(msg_list) >= minlen: | |
727 | raise TypeError("malformed message, must have at least %i elements"%minlen) |
|
731 | raise TypeError("malformed message, must have at least %i elements"%minlen) | |
728 | header = self.unpack(msg_list[1]) |
|
732 | header = self.unpack(msg_list[1]) | |
729 | message['header'] = header |
|
733 | message['header'] = header | |
730 | message['msg_id'] = header['msg_id'] |
|
734 | message['msg_id'] = header['msg_id'] | |
731 | message['msg_type'] = header['msg_type'] |
|
735 | message['msg_type'] = header['msg_type'] | |
732 | message['parent_header'] = self.unpack(msg_list[2]) |
|
736 | message['parent_header'] = self.unpack(msg_list[2]) | |
733 | if content: |
|
737 | if content: | |
734 | message['content'] = self.unpack(msg_list[3]) |
|
738 | message['content'] = self.unpack(msg_list[3]) | |
735 | else: |
|
739 | else: | |
736 | message['content'] = msg_list[3] |
|
740 | message['content'] = msg_list[3] | |
737 |
|
741 | |||
738 | message['buffers'] = msg_list[4:] |
|
742 | message['buffers'] = msg_list[4:] | |
739 | return message |
|
743 | return message | |
740 |
|
744 | |||
741 | def test_msg2obj(): |
|
745 | def test_msg2obj(): | |
742 | am = dict(x=1) |
|
746 | am = dict(x=1) | |
743 | ao = Message(am) |
|
747 | ao = Message(am) | |
744 | assert ao.x == am['x'] |
|
748 | assert ao.x == am['x'] | |
745 |
|
749 | |||
746 | am['y'] = dict(z=1) |
|
750 | am['y'] = dict(z=1) | |
747 | ao = Message(am) |
|
751 | ao = Message(am) | |
748 | assert ao.y.z == am['y']['z'] |
|
752 | assert ao.y.z == am['y']['z'] | |
749 |
|
753 | |||
750 | k1, k2 = 'y', 'z' |
|
754 | k1, k2 = 'y', 'z' | |
751 | assert ao[k1][k2] == am[k1][k2] |
|
755 | assert ao[k1][k2] == am[k1][k2] | |
752 |
|
756 | |||
753 | am2 = dict(ao) |
|
757 | am2 = dict(ao) | |
754 | assert am['x'] == am2['x'] |
|
758 | assert am['x'] == am2['x'] | |
755 | assert am['y']['z'] == am2['y']['z'] |
|
759 | assert am['y']['z'] == am2['y']['z'] | |
756 |
|
760 |
General Comments 0
You need to be logged in to leave comments.
Login now