Show More
@@ -1,223 +1,225 | |||||
1 | """Dependency utilities |
|
1 | """Dependency utilities | |
2 |
|
2 | |||
3 | Authors: |
|
3 | Authors: | |
4 |
|
4 | |||
5 | * Min RK |
|
5 | * Min RK | |
6 | """ |
|
6 | """ | |
7 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
8 | # Copyright (C) 2013 The IPython Development Team |
|
8 | # Copyright (C) 2013 The IPython Development Team | |
9 | # |
|
9 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. | |
12 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
13 |
|
13 | |||
14 | from types import ModuleType |
|
14 | from types import ModuleType | |
15 |
|
15 | |||
16 | from IPython.parallel.client.asyncresult import AsyncResult |
|
16 | from IPython.parallel.client.asyncresult import AsyncResult | |
17 | from IPython.parallel.error import UnmetDependency |
|
17 | from IPython.parallel.error import UnmetDependency | |
18 | from IPython.parallel.util import interactive |
|
18 | from IPython.parallel.util import interactive | |
19 | from IPython.utils import py3compat |
|
19 | from IPython.utils import py3compat | |
20 | from IPython.utils.pickleutil import can, uncan |
|
20 | from IPython.utils.pickleutil import can, uncan | |
21 |
|
21 | |||
22 | class depend(object): |
|
22 | class depend(object): | |
23 | """Dependency decorator, for use with tasks. |
|
23 | """Dependency decorator, for use with tasks. | |
24 |
|
24 | |||
25 | `@depend` lets you define a function for engine dependencies |
|
25 | `@depend` lets you define a function for engine dependencies | |
26 | just like you use `apply` for tasks. |
|
26 | just like you use `apply` for tasks. | |
27 |
|
27 | |||
28 |
|
28 | |||
29 | Examples |
|
29 | Examples | |
30 | -------- |
|
30 | -------- | |
31 | :: |
|
31 | :: | |
32 |
|
32 | |||
33 | @depend(df, a,b, c=5) |
|
33 | @depend(df, a,b, c=5) | |
34 | def f(m,n,p) |
|
34 | def f(m,n,p) | |
35 |
|
35 | |||
36 | view.apply(f, 1,2,3) |
|
36 | view.apply(f, 1,2,3) | |
37 |
|
37 | |||
38 | will call df(a,b,c=5) on the engine, and if it returns False or |
|
38 | will call df(a,b,c=5) on the engine, and if it returns False or | |
39 | raises an UnmetDependency error, then the task will not be run |
|
39 | raises an UnmetDependency error, then the task will not be run | |
40 | and another engine will be tried. |
|
40 | and another engine will be tried. | |
41 | """ |
|
41 | """ | |
42 | def __init__(self, f, *args, **kwargs): |
|
42 | def __init__(self, f, *args, **kwargs): | |
43 | self.f = f |
|
43 | self.f = f | |
44 | self.args = args |
|
44 | self.args = args | |
45 | self.kwargs = kwargs |
|
45 | self.kwargs = kwargs | |
46 |
|
46 | |||
47 | def __call__(self, f): |
|
47 | def __call__(self, f): | |
48 | return dependent(f, self.f, *self.args, **self.kwargs) |
|
48 | return dependent(f, self.f, *self.args, **self.kwargs) | |
49 |
|
49 | |||
50 | class dependent(object): |
|
50 | class dependent(object): | |
51 | """A function that depends on another function. |
|
51 | """A function that depends on another function. | |
52 | This is an object to prevent the closure used |
|
52 | This is an object to prevent the closure used | |
53 | in traditional decorators, which are not picklable. |
|
53 | in traditional decorators, which are not picklable. | |
54 | """ |
|
54 | """ | |
55 |
|
55 | |||
56 | def __init__(self, f, df, *dargs, **dkwargs): |
|
56 | def __init__(self, f, df, *dargs, **dkwargs): | |
57 | self.f = f |
|
57 | self.f = f | |
58 | self.func_name = getattr(f, '__name__', 'f') |
|
58 | self.func_name = getattr(f, '__name__', 'f') | |
59 | self.df = df |
|
59 | self.df = df | |
60 | self.dargs = dargs |
|
60 | self.dargs = dargs | |
61 | self.dkwargs = dkwargs |
|
61 | self.dkwargs = dkwargs | |
62 |
|
62 | |||
63 | def __call__(self, *args, **kwargs): |
|
63 | def check_dependency(self): | |
64 | if self.df(*self.dargs, **self.dkwargs) is False: |
|
64 | if self.df(*self.dargs, **self.dkwargs) is False: | |
65 | raise UnmetDependency() |
|
65 | raise UnmetDependency() | |
|
66 | ||||
|
67 | def __call__(self, *args, **kwargs): | |||
66 | return self.f(*args, **kwargs) |
|
68 | return self.f(*args, **kwargs) | |
67 |
|
69 | |||
68 | if not py3compat.PY3: |
|
70 | if not py3compat.PY3: | |
69 | @property |
|
71 | @property | |
70 | def __name__(self): |
|
72 | def __name__(self): | |
71 | return self.func_name |
|
73 | return self.func_name | |
72 |
|
74 | |||
73 | @interactive |
|
75 | @interactive | |
74 | def _require(*modules, **mapping): |
|
76 | def _require(*modules, **mapping): | |
75 | """Helper for @require decorator.""" |
|
77 | """Helper for @require decorator.""" | |
76 | from IPython.parallel.error import UnmetDependency |
|
78 | from IPython.parallel.error import UnmetDependency | |
77 | from IPython.utils.pickleutil import uncan |
|
79 | from IPython.utils.pickleutil import uncan | |
78 | user_ns = globals() |
|
80 | user_ns = globals() | |
79 | for name in modules: |
|
81 | for name in modules: | |
80 | try: |
|
82 | try: | |
81 | exec 'import %s' % name in user_ns |
|
83 | exec 'import %s' % name in user_ns | |
82 | except ImportError: |
|
84 | except ImportError: | |
83 | raise UnmetDependency(name) |
|
85 | raise UnmetDependency(name) | |
84 |
|
86 | |||
85 | for name, cobj in mapping.items(): |
|
87 | for name, cobj in mapping.items(): | |
86 | user_ns[name] = uncan(cobj, user_ns) |
|
88 | user_ns[name] = uncan(cobj, user_ns) | |
87 | return True |
|
89 | return True | |
88 |
|
90 | |||
89 | def require(*objects, **mapping): |
|
91 | def require(*objects, **mapping): | |
90 | """Simple decorator for requiring local objects and modules to be available |
|
92 | """Simple decorator for requiring local objects and modules to be available | |
91 | when the decorated function is called on the engine. |
|
93 | when the decorated function is called on the engine. | |
92 |
|
94 | |||
93 | Modules specified by name or passed directly will be imported |
|
95 | Modules specified by name or passed directly will be imported | |
94 | prior to calling the decorated function. |
|
96 | prior to calling the decorated function. | |
95 |
|
97 | |||
96 | Objects other than modules will be pushed as a part of the task. |
|
98 | Objects other than modules will be pushed as a part of the task. | |
97 | Functions can be passed positionally, |
|
99 | Functions can be passed positionally, | |
98 | and will be pushed to the engine with their __name__. |
|
100 | and will be pushed to the engine with their __name__. | |
99 | Other objects can be passed by keyword arg. |
|
101 | Other objects can be passed by keyword arg. | |
100 |
|
102 | |||
101 | Examples |
|
103 | Examples | |
102 | -------- |
|
104 | -------- | |
103 |
|
105 | |||
104 | In [1]: @require('numpy') |
|
106 | In [1]: @require('numpy') | |
105 | ...: def norm(a): |
|
107 | ...: def norm(a): | |
106 | ...: return numpy.linalg.norm(a,2) |
|
108 | ...: return numpy.linalg.norm(a,2) | |
107 |
|
109 | |||
108 | In [2]: foo = lambda x: x*x |
|
110 | In [2]: foo = lambda x: x*x | |
109 | In [3]: @require(foo) |
|
111 | In [3]: @require(foo) | |
110 | ...: def bar(a): |
|
112 | ...: def bar(a): | |
111 | ...: return foo(1-a) |
|
113 | ...: return foo(1-a) | |
112 | """ |
|
114 | """ | |
113 | names = [] |
|
115 | names = [] | |
114 | for obj in objects: |
|
116 | for obj in objects: | |
115 | if isinstance(obj, ModuleType): |
|
117 | if isinstance(obj, ModuleType): | |
116 | obj = obj.__name__ |
|
118 | obj = obj.__name__ | |
117 |
|
119 | |||
118 | if isinstance(obj, basestring): |
|
120 | if isinstance(obj, basestring): | |
119 | names.append(obj) |
|
121 | names.append(obj) | |
120 | elif hasattr(obj, '__name__'): |
|
122 | elif hasattr(obj, '__name__'): | |
121 | mapping[obj.__name__] = obj |
|
123 | mapping[obj.__name__] = obj | |
122 | else: |
|
124 | else: | |
123 | raise TypeError("Objects other than modules and functions " |
|
125 | raise TypeError("Objects other than modules and functions " | |
124 | "must be passed by kwarg, but got: %s" % type(obj) |
|
126 | "must be passed by kwarg, but got: %s" % type(obj) | |
125 | ) |
|
127 | ) | |
126 |
|
128 | |||
127 | for name, obj in mapping.items(): |
|
129 | for name, obj in mapping.items(): | |
128 | mapping[name] = can(obj) |
|
130 | mapping[name] = can(obj) | |
129 | return depend(_require, *names, **mapping) |
|
131 | return depend(_require, *names, **mapping) | |
130 |
|
132 | |||
131 | class Dependency(set): |
|
133 | class Dependency(set): | |
132 | """An object for representing a set of msg_id dependencies. |
|
134 | """An object for representing a set of msg_id dependencies. | |
133 |
|
135 | |||
134 | Subclassed from set(). |
|
136 | Subclassed from set(). | |
135 |
|
137 | |||
136 | Parameters |
|
138 | Parameters | |
137 | ---------- |
|
139 | ---------- | |
138 | dependencies: list/set of msg_ids or AsyncResult objects or output of Dependency.as_dict() |
|
140 | dependencies: list/set of msg_ids or AsyncResult objects or output of Dependency.as_dict() | |
139 | The msg_ids to depend on |
|
141 | The msg_ids to depend on | |
140 | all : bool [default True] |
|
142 | all : bool [default True] | |
141 | Whether the dependency should be considered met when *all* depending tasks have completed |
|
143 | Whether the dependency should be considered met when *all* depending tasks have completed | |
142 | or only when *any* have been completed. |
|
144 | or only when *any* have been completed. | |
143 | success : bool [default True] |
|
145 | success : bool [default True] | |
144 | Whether to consider successes as fulfilling dependencies. |
|
146 | Whether to consider successes as fulfilling dependencies. | |
145 | failure : bool [default False] |
|
147 | failure : bool [default False] | |
146 | Whether to consider failures as fulfilling dependencies. |
|
148 | Whether to consider failures as fulfilling dependencies. | |
147 |
|
149 | |||
148 | If `all=success=True` and `failure=False`, then the task will fail with an ImpossibleDependency |
|
150 | If `all=success=True` and `failure=False`, then the task will fail with an ImpossibleDependency | |
149 | as soon as the first depended-upon task fails. |
|
151 | as soon as the first depended-upon task fails. | |
150 | """ |
|
152 | """ | |
151 |
|
153 | |||
152 | all=True |
|
154 | all=True | |
153 | success=True |
|
155 | success=True | |
154 | failure=True |
|
156 | failure=True | |
155 |
|
157 | |||
156 | def __init__(self, dependencies=[], all=True, success=True, failure=False): |
|
158 | def __init__(self, dependencies=[], all=True, success=True, failure=False): | |
157 | if isinstance(dependencies, dict): |
|
159 | if isinstance(dependencies, dict): | |
158 | # load from dict |
|
160 | # load from dict | |
159 | all = dependencies.get('all', True) |
|
161 | all = dependencies.get('all', True) | |
160 | success = dependencies.get('success', success) |
|
162 | success = dependencies.get('success', success) | |
161 | failure = dependencies.get('failure', failure) |
|
163 | failure = dependencies.get('failure', failure) | |
162 | dependencies = dependencies.get('dependencies', []) |
|
164 | dependencies = dependencies.get('dependencies', []) | |
163 | ids = [] |
|
165 | ids = [] | |
164 |
|
166 | |||
165 | # extract ids from various sources: |
|
167 | # extract ids from various sources: | |
166 | if isinstance(dependencies, (basestring, AsyncResult)): |
|
168 | if isinstance(dependencies, (basestring, AsyncResult)): | |
167 | dependencies = [dependencies] |
|
169 | dependencies = [dependencies] | |
168 | for d in dependencies: |
|
170 | for d in dependencies: | |
169 | if isinstance(d, basestring): |
|
171 | if isinstance(d, basestring): | |
170 | ids.append(d) |
|
172 | ids.append(d) | |
171 | elif isinstance(d, AsyncResult): |
|
173 | elif isinstance(d, AsyncResult): | |
172 | ids.extend(d.msg_ids) |
|
174 | ids.extend(d.msg_ids) | |
173 | else: |
|
175 | else: | |
174 | raise TypeError("invalid dependency type: %r"%type(d)) |
|
176 | raise TypeError("invalid dependency type: %r"%type(d)) | |
175 |
|
177 | |||
176 | set.__init__(self, ids) |
|
178 | set.__init__(self, ids) | |
177 | self.all = all |
|
179 | self.all = all | |
178 | if not (success or failure): |
|
180 | if not (success or failure): | |
179 | raise ValueError("Must depend on at least one of successes or failures!") |
|
181 | raise ValueError("Must depend on at least one of successes or failures!") | |
180 | self.success=success |
|
182 | self.success=success | |
181 | self.failure = failure |
|
183 | self.failure = failure | |
182 |
|
184 | |||
183 | def check(self, completed, failed=None): |
|
185 | def check(self, completed, failed=None): | |
184 | """check whether our dependencies have been met.""" |
|
186 | """check whether our dependencies have been met.""" | |
185 | if len(self) == 0: |
|
187 | if len(self) == 0: | |
186 | return True |
|
188 | return True | |
187 | against = set() |
|
189 | against = set() | |
188 | if self.success: |
|
190 | if self.success: | |
189 | against = completed |
|
191 | against = completed | |
190 | if failed is not None and self.failure: |
|
192 | if failed is not None and self.failure: | |
191 | against = against.union(failed) |
|
193 | against = against.union(failed) | |
192 | if self.all: |
|
194 | if self.all: | |
193 | return self.issubset(against) |
|
195 | return self.issubset(against) | |
194 | else: |
|
196 | else: | |
195 | return not self.isdisjoint(against) |
|
197 | return not self.isdisjoint(against) | |
196 |
|
198 | |||
197 | def unreachable(self, completed, failed=None): |
|
199 | def unreachable(self, completed, failed=None): | |
198 | """return whether this dependency has become impossible.""" |
|
200 | """return whether this dependency has become impossible.""" | |
199 | if len(self) == 0: |
|
201 | if len(self) == 0: | |
200 | return False |
|
202 | return False | |
201 | against = set() |
|
203 | against = set() | |
202 | if not self.success: |
|
204 | if not self.success: | |
203 | against = completed |
|
205 | against = completed | |
204 | if failed is not None and not self.failure: |
|
206 | if failed is not None and not self.failure: | |
205 | against = against.union(failed) |
|
207 | against = against.union(failed) | |
206 | if self.all: |
|
208 | if self.all: | |
207 | return not self.isdisjoint(against) |
|
209 | return not self.isdisjoint(against) | |
208 | else: |
|
210 | else: | |
209 | return self.issubset(against) |
|
211 | return self.issubset(against) | |
210 |
|
212 | |||
211 |
|
213 | |||
212 | def as_dict(self): |
|
214 | def as_dict(self): | |
213 | """Represent this dependency as a dict. For json compatibility.""" |
|
215 | """Represent this dependency as a dict. For json compatibility.""" | |
214 | return dict( |
|
216 | return dict( | |
215 | dependencies=list(self), |
|
217 | dependencies=list(self), | |
216 | all=self.all, |
|
218 | all=self.all, | |
217 | success=self.success, |
|
219 | success=self.success, | |
218 | failure=self.failure |
|
220 | failure=self.failure | |
219 | ) |
|
221 | ) | |
220 |
|
222 | |||
221 |
|
223 | |||
222 | __all__ = ['depend', 'require', 'dependent', 'Dependency'] |
|
224 | __all__ = ['depend', 'require', 'dependent', 'Dependency'] | |
223 |
|
225 |
@@ -1,333 +1,338 | |||||
1 | # encoding: utf-8 |
|
1 | # encoding: utf-8 | |
2 |
|
2 | |||
3 | """Pickle related utilities. Perhaps this should be called 'can'.""" |
|
3 | """Pickle related utilities. Perhaps this should be called 'can'.""" | |
4 |
|
4 | |||
5 | __docformat__ = "restructuredtext en" |
|
5 | __docformat__ = "restructuredtext en" | |
6 |
|
6 | |||
7 | #------------------------------------------------------------------------------- |
|
7 | #------------------------------------------------------------------------------- | |
8 | # Copyright (C) 2008-2011 The IPython Development Team |
|
8 | # Copyright (C) 2008-2011 The IPython Development Team | |
9 | # |
|
9 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. | |
12 | #------------------------------------------------------------------------------- |
|
12 | #------------------------------------------------------------------------------- | |
13 |
|
13 | |||
14 | #------------------------------------------------------------------------------- |
|
14 | #------------------------------------------------------------------------------- | |
15 | # Imports |
|
15 | # Imports | |
16 | #------------------------------------------------------------------------------- |
|
16 | #------------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | import copy |
|
18 | import copy | |
19 | import logging |
|
19 | import logging | |
20 | import sys |
|
20 | import sys | |
21 | from types import FunctionType |
|
21 | from types import FunctionType | |
22 |
|
22 | |||
23 | try: |
|
23 | try: | |
24 | import cPickle as pickle |
|
24 | import cPickle as pickle | |
25 | except ImportError: |
|
25 | except ImportError: | |
26 | import pickle |
|
26 | import pickle | |
27 |
|
27 | |||
28 | try: |
|
28 | try: | |
29 | import numpy |
|
29 | import numpy | |
30 | except: |
|
30 | except: | |
31 | numpy = None |
|
31 | numpy = None | |
32 |
|
32 | |||
33 | import codeutil |
|
33 | import codeutil | |
34 | import py3compat |
|
34 | import py3compat | |
35 | from importstring import import_item |
|
35 | from importstring import import_item | |
36 |
|
36 | |||
37 | from IPython.config import Application |
|
37 | from IPython.config import Application | |
38 |
|
38 | |||
39 | if py3compat.PY3: |
|
39 | if py3compat.PY3: | |
40 | buffer = memoryview |
|
40 | buffer = memoryview | |
41 | class_type = type |
|
41 | class_type = type | |
42 | else: |
|
42 | else: | |
43 | from types import ClassType |
|
43 | from types import ClassType | |
44 | class_type = (type, ClassType) |
|
44 | class_type = (type, ClassType) | |
45 |
|
45 | |||
46 | #------------------------------------------------------------------------------- |
|
46 | #------------------------------------------------------------------------------- | |
47 | # Classes |
|
47 | # Classes | |
48 | #------------------------------------------------------------------------------- |
|
48 | #------------------------------------------------------------------------------- | |
49 |
|
49 | |||
50 |
|
50 | |||
51 | class CannedObject(object): |
|
51 | class CannedObject(object): | |
52 | def __init__(self, obj, keys=[], hook=None): |
|
52 | def __init__(self, obj, keys=[], hook=None): | |
53 | self.keys = keys |
|
53 | self.keys = keys | |
54 | self.obj = copy.copy(obj) |
|
54 | self.obj = copy.copy(obj) | |
55 | self.hook = can(hook) |
|
55 | self.hook = can(hook) | |
56 | for key in keys: |
|
56 | for key in keys: | |
57 | setattr(self.obj, key, can(getattr(obj, key))) |
|
57 | setattr(self.obj, key, can(getattr(obj, key))) | |
58 |
|
58 | |||
59 | self.buffers = [] |
|
59 | self.buffers = [] | |
60 |
|
60 | |||
61 | def get_object(self, g=None): |
|
61 | def get_object(self, g=None): | |
62 | if g is None: |
|
62 | if g is None: | |
63 | g = {} |
|
63 | g = {} | |
64 | obj = self.obj |
|
64 | obj = self.obj | |
65 | for key in self.keys: |
|
65 | for key in self.keys: | |
66 | setattr(obj, key, uncan(getattr(obj, key), g)) |
|
66 | setattr(obj, key, uncan(getattr(obj, key), g)) | |
67 |
|
67 | |||
68 | if self.hook: |
|
68 | if self.hook: | |
69 | self.hook = uncan(self.hook, g) |
|
69 | self.hook = uncan(self.hook, g) | |
70 | self.hook(obj, g) |
|
70 | self.hook(obj, g) | |
71 | return self.obj |
|
71 | return self.obj | |
72 |
|
72 | |||
73 |
|
73 | |||
74 | class Reference(CannedObject): |
|
74 | class Reference(CannedObject): | |
75 | """object for wrapping a remote reference by name.""" |
|
75 | """object for wrapping a remote reference by name.""" | |
76 | def __init__(self, name): |
|
76 | def __init__(self, name): | |
77 | if not isinstance(name, basestring): |
|
77 | if not isinstance(name, basestring): | |
78 | raise TypeError("illegal name: %r"%name) |
|
78 | raise TypeError("illegal name: %r"%name) | |
79 | self.name = name |
|
79 | self.name = name | |
80 | self.buffers = [] |
|
80 | self.buffers = [] | |
81 |
|
81 | |||
82 | def __repr__(self): |
|
82 | def __repr__(self): | |
83 | return "<Reference: %r>"%self.name |
|
83 | return "<Reference: %r>"%self.name | |
84 |
|
84 | |||
85 | def get_object(self, g=None): |
|
85 | def get_object(self, g=None): | |
86 | if g is None: |
|
86 | if g is None: | |
87 | g = {} |
|
87 | g = {} | |
88 |
|
88 | |||
89 | return eval(self.name, g) |
|
89 | return eval(self.name, g) | |
90 |
|
90 | |||
91 |
|
91 | |||
92 | class CannedFunction(CannedObject): |
|
92 | class CannedFunction(CannedObject): | |
93 |
|
93 | |||
94 | def __init__(self, f): |
|
94 | def __init__(self, f): | |
95 | self._check_type(f) |
|
95 | self._check_type(f) | |
96 | self.code = f.func_code |
|
96 | self.code = f.func_code | |
97 | if f.func_defaults: |
|
97 | if f.func_defaults: | |
98 | self.defaults = [ can(fd) for fd in f.func_defaults ] |
|
98 | self.defaults = [ can(fd) for fd in f.func_defaults ] | |
99 | else: |
|
99 | else: | |
100 | self.defaults = None |
|
100 | self.defaults = None | |
101 | self.module = f.__module__ or '__main__' |
|
101 | self.module = f.__module__ or '__main__' | |
102 | self.__name__ = f.__name__ |
|
102 | self.__name__ = f.__name__ | |
103 | self.buffers = [] |
|
103 | self.buffers = [] | |
104 |
|
104 | |||
105 | def _check_type(self, obj): |
|
105 | def _check_type(self, obj): | |
106 | assert isinstance(obj, FunctionType), "Not a function type" |
|
106 | assert isinstance(obj, FunctionType), "Not a function type" | |
107 |
|
107 | |||
108 | def get_object(self, g=None): |
|
108 | def get_object(self, g=None): | |
109 | # try to load function back into its module: |
|
109 | # try to load function back into its module: | |
110 | if not self.module.startswith('__'): |
|
110 | if not self.module.startswith('__'): | |
111 | __import__(self.module) |
|
111 | __import__(self.module) | |
112 | g = sys.modules[self.module].__dict__ |
|
112 | g = sys.modules[self.module].__dict__ | |
113 |
|
113 | |||
114 | if g is None: |
|
114 | if g is None: | |
115 | g = {} |
|
115 | g = {} | |
116 | if self.defaults: |
|
116 | if self.defaults: | |
117 | defaults = tuple(uncan(cfd, g) for cfd in self.defaults) |
|
117 | defaults = tuple(uncan(cfd, g) for cfd in self.defaults) | |
118 | else: |
|
118 | else: | |
119 | defaults = None |
|
119 | defaults = None | |
120 | newFunc = FunctionType(self.code, g, self.__name__, defaults) |
|
120 | newFunc = FunctionType(self.code, g, self.__name__, defaults) | |
121 | return newFunc |
|
121 | return newFunc | |
122 |
|
122 | |||
123 | class CannedClass(CannedObject): |
|
123 | class CannedClass(CannedObject): | |
124 |
|
124 | |||
125 | def __init__(self, cls): |
|
125 | def __init__(self, cls): | |
126 | self._check_type(cls) |
|
126 | self._check_type(cls) | |
127 | self.name = cls.__name__ |
|
127 | self.name = cls.__name__ | |
128 | self.old_style = not isinstance(cls, type) |
|
128 | self.old_style = not isinstance(cls, type) | |
129 | self._canned_dict = {} |
|
129 | self._canned_dict = {} | |
130 | for k,v in cls.__dict__.items(): |
|
130 | for k,v in cls.__dict__.items(): | |
131 | if k not in ('__weakref__', '__dict__'): |
|
131 | if k not in ('__weakref__', '__dict__'): | |
132 | self._canned_dict[k] = can(v) |
|
132 | self._canned_dict[k] = can(v) | |
133 | if self.old_style: |
|
133 | if self.old_style: | |
134 | mro = [] |
|
134 | mro = [] | |
135 | else: |
|
135 | else: | |
136 | mro = cls.mro() |
|
136 | mro = cls.mro() | |
137 |
|
137 | |||
138 | self.parents = [ can(c) for c in mro[1:] ] |
|
138 | self.parents = [ can(c) for c in mro[1:] ] | |
139 | self.buffers = [] |
|
139 | self.buffers = [] | |
140 |
|
140 | |||
141 | def _check_type(self, obj): |
|
141 | def _check_type(self, obj): | |
142 | assert isinstance(obj, class_type), "Not a class type" |
|
142 | assert isinstance(obj, class_type), "Not a class type" | |
143 |
|
143 | |||
144 | def get_object(self, g=None): |
|
144 | def get_object(self, g=None): | |
145 | parents = tuple(uncan(p, g) for p in self.parents) |
|
145 | parents = tuple(uncan(p, g) for p in self.parents) | |
146 | return type(self.name, parents, uncan_dict(self._canned_dict, g=g)) |
|
146 | return type(self.name, parents, uncan_dict(self._canned_dict, g=g)) | |
147 |
|
147 | |||
148 | class CannedArray(CannedObject): |
|
148 | class CannedArray(CannedObject): | |
149 | def __init__(self, obj): |
|
149 | def __init__(self, obj): | |
150 | self.shape = obj.shape |
|
150 | self.shape = obj.shape | |
151 | self.dtype = obj.dtype.descr if obj.dtype.fields else obj.dtype.str |
|
151 | self.dtype = obj.dtype.descr if obj.dtype.fields else obj.dtype.str | |
152 | if sum(obj.shape) == 0: |
|
152 | if sum(obj.shape) == 0: | |
153 | # just pickle it |
|
153 | # just pickle it | |
154 | self.buffers = [pickle.dumps(obj, -1)] |
|
154 | self.buffers = [pickle.dumps(obj, -1)] | |
155 | else: |
|
155 | else: | |
156 | # ensure contiguous |
|
156 | # ensure contiguous | |
157 | obj = numpy.ascontiguousarray(obj, dtype=None) |
|
157 | obj = numpy.ascontiguousarray(obj, dtype=None) | |
158 | self.buffers = [buffer(obj)] |
|
158 | self.buffers = [buffer(obj)] | |
159 |
|
159 | |||
160 | def get_object(self, g=None): |
|
160 | def get_object(self, g=None): | |
161 | data = self.buffers[0] |
|
161 | data = self.buffers[0] | |
162 | if sum(self.shape) == 0: |
|
162 | if sum(self.shape) == 0: | |
163 | # no shape, we just pickled it |
|
163 | # no shape, we just pickled it | |
164 | return pickle.loads(data) |
|
164 | return pickle.loads(data) | |
165 | else: |
|
165 | else: | |
166 | return numpy.frombuffer(data, dtype=self.dtype).reshape(self.shape) |
|
166 | return numpy.frombuffer(data, dtype=self.dtype).reshape(self.shape) | |
167 |
|
167 | |||
168 |
|
168 | |||
169 | class CannedBytes(CannedObject): |
|
169 | class CannedBytes(CannedObject): | |
170 | wrap = bytes |
|
170 | wrap = bytes | |
171 | def __init__(self, obj): |
|
171 | def __init__(self, obj): | |
172 | self.buffers = [obj] |
|
172 | self.buffers = [obj] | |
173 |
|
173 | |||
174 | def get_object(self, g=None): |
|
174 | def get_object(self, g=None): | |
175 | data = self.buffers[0] |
|
175 | data = self.buffers[0] | |
176 | return self.wrap(data) |
|
176 | return self.wrap(data) | |
177 |
|
177 | |||
178 | def CannedBuffer(CannedBytes): |
|
178 | def CannedBuffer(CannedBytes): | |
179 | wrap = buffer |
|
179 | wrap = buffer | |
180 |
|
180 | |||
181 | #------------------------------------------------------------------------------- |
|
181 | #------------------------------------------------------------------------------- | |
182 | # Functions |
|
182 | # Functions | |
183 | #------------------------------------------------------------------------------- |
|
183 | #------------------------------------------------------------------------------- | |
184 |
|
184 | |||
185 | def _logger(): |
|
185 | def _logger(): | |
186 | """get the logger for the current Application |
|
186 | """get the logger for the current Application | |
187 |
|
187 | |||
188 | the root logger will be used if no Application is running |
|
188 | the root logger will be used if no Application is running | |
189 | """ |
|
189 | """ | |
190 | if Application.initialized(): |
|
190 | if Application.initialized(): | |
191 | logger = Application.instance().log |
|
191 | logger = Application.instance().log | |
192 | else: |
|
192 | else: | |
193 | logger = logging.getLogger() |
|
193 | logger = logging.getLogger() | |
194 | if not logger.handlers: |
|
194 | if not logger.handlers: | |
195 | logging.basicConfig() |
|
195 | logging.basicConfig() | |
196 |
|
196 | |||
197 | return logger |
|
197 | return logger | |
198 |
|
198 | |||
199 | def _import_mapping(mapping, original=None): |
|
199 | def _import_mapping(mapping, original=None): | |
200 | """import any string-keys in a type mapping |
|
200 | """import any string-keys in a type mapping | |
201 |
|
201 | |||
202 | """ |
|
202 | """ | |
203 | log = _logger() |
|
203 | log = _logger() | |
204 | log.debug("Importing canning map") |
|
204 | log.debug("Importing canning map") | |
205 | for key,value in mapping.items(): |
|
205 | for key,value in mapping.items(): | |
206 | if isinstance(key, basestring): |
|
206 | if isinstance(key, basestring): | |
207 | try: |
|
207 | try: | |
208 | cls = import_item(key) |
|
208 | cls = import_item(key) | |
209 | except Exception: |
|
209 | except Exception: | |
210 | if original and key not in original: |
|
210 | if original and key not in original: | |
211 | # only message on user-added classes |
|
211 | # only message on user-added classes | |
212 | log.error("cannning class not importable: %r", key, exc_info=True) |
|
212 | log.error("cannning class not importable: %r", key, exc_info=True) | |
213 | mapping.pop(key) |
|
213 | mapping.pop(key) | |
214 | else: |
|
214 | else: | |
215 | mapping[cls] = mapping.pop(key) |
|
215 | mapping[cls] = mapping.pop(key) | |
216 |
|
216 | |||
217 | def istype(obj, check): |
|
217 | def istype(obj, check): | |
218 | """like isinstance(obj, check), but strict |
|
218 | """like isinstance(obj, check), but strict | |
219 |
|
219 | |||
220 | This won't catch subclasses. |
|
220 | This won't catch subclasses. | |
221 | """ |
|
221 | """ | |
222 | if isinstance(check, tuple): |
|
222 | if isinstance(check, tuple): | |
223 | for cls in check: |
|
223 | for cls in check: | |
224 | if type(obj) is cls: |
|
224 | if type(obj) is cls: | |
225 | return True |
|
225 | return True | |
226 | return False |
|
226 | return False | |
227 | else: |
|
227 | else: | |
228 | return type(obj) is check |
|
228 | return type(obj) is check | |
229 |
|
229 | |||
230 | def can(obj): |
|
230 | def can(obj): | |
231 | """prepare an object for pickling""" |
|
231 | """prepare an object for pickling""" | |
232 |
|
232 | |||
233 | import_needed = False |
|
233 | import_needed = False | |
234 |
|
234 | |||
235 | for cls,canner in can_map.iteritems(): |
|
235 | for cls,canner in can_map.iteritems(): | |
236 | if isinstance(cls, basestring): |
|
236 | if isinstance(cls, basestring): | |
237 | import_needed = True |
|
237 | import_needed = True | |
238 | break |
|
238 | break | |
239 | elif istype(obj, cls): |
|
239 | elif istype(obj, cls): | |
240 | return canner(obj) |
|
240 | return canner(obj) | |
241 |
|
241 | |||
242 | if import_needed: |
|
242 | if import_needed: | |
243 | # perform can_map imports, then try again |
|
243 | # perform can_map imports, then try again | |
244 | # this will usually only happen once |
|
244 | # this will usually only happen once | |
245 | _import_mapping(can_map, _original_can_map) |
|
245 | _import_mapping(can_map, _original_can_map) | |
246 | return can(obj) |
|
246 | return can(obj) | |
247 |
|
247 | |||
248 | return obj |
|
248 | return obj | |
249 |
|
249 | |||
250 | def can_class(obj): |
|
250 | def can_class(obj): | |
251 | if isinstance(obj, class_type) and obj.__module__ == '__main__': |
|
251 | if isinstance(obj, class_type) and obj.__module__ == '__main__': | |
252 | return CannedClass(obj) |
|
252 | return CannedClass(obj) | |
253 | else: |
|
253 | else: | |
254 | return obj |
|
254 | return obj | |
255 |
|
255 | |||
256 | def can_dict(obj): |
|
256 | def can_dict(obj): | |
257 | """can the *values* of a dict""" |
|
257 | """can the *values* of a dict""" | |
258 | if istype(obj, dict): |
|
258 | if istype(obj, dict): | |
259 | newobj = {} |
|
259 | newobj = {} | |
260 | for k, v in obj.iteritems(): |
|
260 | for k, v in obj.iteritems(): | |
261 | newobj[k] = can(v) |
|
261 | newobj[k] = can(v) | |
262 | return newobj |
|
262 | return newobj | |
263 | else: |
|
263 | else: | |
264 | return obj |
|
264 | return obj | |
265 |
|
265 | |||
266 | sequence_types = (list, tuple, set) |
|
266 | sequence_types = (list, tuple, set) | |
267 |
|
267 | |||
268 | def can_sequence(obj): |
|
268 | def can_sequence(obj): | |
269 | """can the elements of a sequence""" |
|
269 | """can the elements of a sequence""" | |
270 | if istype(obj, sequence_types): |
|
270 | if istype(obj, sequence_types): | |
271 | t = type(obj) |
|
271 | t = type(obj) | |
272 | return t([can(i) for i in obj]) |
|
272 | return t([can(i) for i in obj]) | |
273 | else: |
|
273 | else: | |
274 | return obj |
|
274 | return obj | |
275 |
|
275 | |||
276 | def uncan(obj, g=None): |
|
276 | def uncan(obj, g=None): | |
277 | """invert canning""" |
|
277 | """invert canning""" | |
278 |
|
278 | |||
279 | import_needed = False |
|
279 | import_needed = False | |
280 | for cls,uncanner in uncan_map.iteritems(): |
|
280 | for cls,uncanner in uncan_map.iteritems(): | |
281 | if isinstance(cls, basestring): |
|
281 | if isinstance(cls, basestring): | |
282 | import_needed = True |
|
282 | import_needed = True | |
283 | break |
|
283 | break | |
284 | elif isinstance(obj, cls): |
|
284 | elif isinstance(obj, cls): | |
285 | return uncanner(obj, g) |
|
285 | return uncanner(obj, g) | |
286 |
|
286 | |||
287 | if import_needed: |
|
287 | if import_needed: | |
288 | # perform uncan_map imports, then try again |
|
288 | # perform uncan_map imports, then try again | |
289 | # this will usually only happen once |
|
289 | # this will usually only happen once | |
290 | _import_mapping(uncan_map, _original_uncan_map) |
|
290 | _import_mapping(uncan_map, _original_uncan_map) | |
291 | return uncan(obj, g) |
|
291 | return uncan(obj, g) | |
292 |
|
292 | |||
293 | return obj |
|
293 | return obj | |
294 |
|
294 | |||
295 | def uncan_dict(obj, g=None): |
|
295 | def uncan_dict(obj, g=None): | |
296 | if istype(obj, dict): |
|
296 | if istype(obj, dict): | |
297 | newobj = {} |
|
297 | newobj = {} | |
298 | for k, v in obj.iteritems(): |
|
298 | for k, v in obj.iteritems(): | |
299 | newobj[k] = uncan(v,g) |
|
299 | newobj[k] = uncan(v,g) | |
300 | return newobj |
|
300 | return newobj | |
301 | else: |
|
301 | else: | |
302 | return obj |
|
302 | return obj | |
303 |
|
303 | |||
304 | def uncan_sequence(obj, g=None): |
|
304 | def uncan_sequence(obj, g=None): | |
305 | if istype(obj, sequence_types): |
|
305 | if istype(obj, sequence_types): | |
306 | t = type(obj) |
|
306 | t = type(obj) | |
307 | return t([uncan(i,g) for i in obj]) |
|
307 | return t([uncan(i,g) for i in obj]) | |
308 | else: |
|
308 | else: | |
309 | return obj |
|
309 | return obj | |
310 |
|
310 | |||
|
311 | def _uncan_dependent_hook(dep, g=None): | |||
|
312 | dep.check_dependency() | |||
|
313 | ||||
|
314 | def can_dependent(obj): | |||
|
315 | return CannedObject(obj, keys=('f', 'df'), hook=_uncan_dependent_hook) | |||
311 |
|
316 | |||
312 | #------------------------------------------------------------------------------- |
|
317 | #------------------------------------------------------------------------------- | |
313 | # API dictionaries |
|
318 | # API dictionaries | |
314 | #------------------------------------------------------------------------------- |
|
319 | #------------------------------------------------------------------------------- | |
315 |
|
320 | |||
316 | # These dicts can be extended for custom serialization of new objects |
|
321 | # These dicts can be extended for custom serialization of new objects | |
317 |
|
322 | |||
318 | can_map = { |
|
323 | can_map = { | |
319 |
'IPython.parallel.dependent' : |
|
324 | 'IPython.parallel.dependent' : can_dependent, | |
320 | 'numpy.ndarray' : CannedArray, |
|
325 | 'numpy.ndarray' : CannedArray, | |
321 | FunctionType : CannedFunction, |
|
326 | FunctionType : CannedFunction, | |
322 | bytes : CannedBytes, |
|
327 | bytes : CannedBytes, | |
323 | buffer : CannedBuffer, |
|
328 | buffer : CannedBuffer, | |
324 | class_type : can_class, |
|
329 | class_type : can_class, | |
325 | } |
|
330 | } | |
326 |
|
331 | |||
327 | uncan_map = { |
|
332 | uncan_map = { | |
328 | CannedObject : lambda obj, g: obj.get_object(g), |
|
333 | CannedObject : lambda obj, g: obj.get_object(g), | |
329 | } |
|
334 | } | |
330 |
|
335 | |||
331 | # for use in _import_mapping: |
|
336 | # for use in _import_mapping: | |
332 | _original_can_map = can_map.copy() |
|
337 | _original_can_map = can_map.copy() | |
333 | _original_uncan_map = uncan_map.copy() |
|
338 | _original_uncan_map = uncan_map.copy() |
General Comments 0
You need to be logged in to leave comments.
Login now