##// END OF EJS Templates
don't use common names in require decorators...
MinRK -
Show More
@@ -1,229 +1,229 b''
1 1 """Dependency utilities
2 2
3 3 Authors:
4 4
5 5 * Min RK
6 6 """
7 7 #-----------------------------------------------------------------------------
8 8 # Copyright (C) 2013 The IPython Development Team
9 9 #
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 from types import ModuleType
15 15
16 16 from IPython.parallel.client.asyncresult import AsyncResult
17 17 from IPython.parallel.error import UnmetDependency
18 18 from IPython.parallel.util import interactive
19 19 from IPython.utils import py3compat
20 20 from IPython.utils.py3compat import string_types
21 21 from IPython.utils.pickleutil import can, uncan
22 22
23 23 class depend(object):
24 24 """Dependency decorator, for use with tasks.
25 25
26 26 `@depend` lets you define a function for engine dependencies
27 27 just like you use `apply` for tasks.
28 28
29 29
30 30 Examples
31 31 --------
32 32 ::
33 33
34 34 @depend(df, a,b, c=5)
35 35 def f(m,n,p)
36 36
37 37 view.apply(f, 1,2,3)
38 38
39 39 will call df(a,b,c=5) on the engine, and if it returns False or
40 40 raises an UnmetDependency error, then the task will not be run
41 41 and another engine will be tried.
42 42 """
43 def __init__(self, f, *args, **kwargs):
44 self.f = f
43 def __init__(self, _wrapped_f, *args, **kwargs):
44 self.f = _wrapped_f
45 45 self.args = args
46 46 self.kwargs = kwargs
47 47
48 48 def __call__(self, f):
49 49 return dependent(f, self.f, *self.args, **self.kwargs)
50 50
51 51 class dependent(object):
52 52 """A function that depends on another function.
53 53 This is an object to prevent the closure used
54 54 in traditional decorators, which are not picklable.
55 55 """
56 56
57 def __init__(self, f, df, *dargs, **dkwargs):
58 self.f = f
57 def __init__(self, _wrapped_f, _wrapped_df, *dargs, **dkwargs):
58 self.f = _wrapped_f
59 59 name = getattr(f, '__name__', 'f')
60 60 if py3compat.PY3:
61 61 self.__name__ = name
62 62 else:
63 63 self.func_name = name
64 self.df = df
64 self.df = _wrapped_df
65 65 self.dargs = dargs
66 66 self.dkwargs = dkwargs
67 67
68 68 def check_dependency(self):
69 69 if self.df(*self.dargs, **self.dkwargs) is False:
70 70 raise UnmetDependency()
71 71
72 72 def __call__(self, *args, **kwargs):
73 73 return self.f(*args, **kwargs)
74 74
75 75 if not py3compat.PY3:
76 76 @property
77 77 def __name__(self):
78 78 return self.func_name
79 79
80 80 @interactive
81 81 def _require(*modules, **mapping):
82 82 """Helper for @require decorator."""
83 83 from IPython.parallel.error import UnmetDependency
84 84 from IPython.utils.pickleutil import uncan
85 85 user_ns = globals()
86 86 for name in modules:
87 87 try:
88 88 exec('import %s' % name, user_ns)
89 89 except ImportError:
90 90 raise UnmetDependency(name)
91 91
92 92 for name, cobj in mapping.items():
93 93 user_ns[name] = uncan(cobj, user_ns)
94 94 return True
95 95
96 96 def require(*objects, **mapping):
97 97 """Simple decorator for requiring local objects and modules to be available
98 98 when the decorated function is called on the engine.
99 99
100 100 Modules specified by name or passed directly will be imported
101 101 prior to calling the decorated function.
102 102
103 103 Objects other than modules will be pushed as a part of the task.
104 104 Functions can be passed positionally,
105 105 and will be pushed to the engine with their __name__.
106 106 Other objects can be passed by keyword arg.
107 107
108 108 Examples::
109 109
110 110 In [1]: @require('numpy')
111 111 ...: def norm(a):
112 112 ...: return numpy.linalg.norm(a,2)
113 113
114 114 In [2]: foo = lambda x: x*x
115 115 In [3]: @require(foo)
116 116 ...: def bar(a):
117 117 ...: return foo(1-a)
118 118 """
119 119 names = []
120 120 for obj in objects:
121 121 if isinstance(obj, ModuleType):
122 122 obj = obj.__name__
123 123
124 124 if isinstance(obj, string_types):
125 125 names.append(obj)
126 126 elif hasattr(obj, '__name__'):
127 127 mapping[obj.__name__] = obj
128 128 else:
129 129 raise TypeError("Objects other than modules and functions "
130 130 "must be passed by kwarg, but got: %s" % type(obj)
131 131 )
132 132
133 133 for name, obj in mapping.items():
134 134 mapping[name] = can(obj)
135 135 return depend(_require, *names, **mapping)
136 136
137 137 class Dependency(set):
138 138 """An object for representing a set of msg_id dependencies.
139 139
140 140 Subclassed from set().
141 141
142 142 Parameters
143 143 ----------
144 144 dependencies: list/set of msg_ids or AsyncResult objects or output of Dependency.as_dict()
145 145 The msg_ids to depend on
146 146 all : bool [default True]
147 147 Whether the dependency should be considered met when *all* depending tasks have completed
148 148 or only when *any* have been completed.
149 149 success : bool [default True]
150 150 Whether to consider successes as fulfilling dependencies.
151 151 failure : bool [default False]
152 152 Whether to consider failures as fulfilling dependencies.
153 153
154 154 If `all=success=True` and `failure=False`, then the task will fail with an ImpossibleDependency
155 155 as soon as the first depended-upon task fails.
156 156 """
157 157
158 158 all=True
159 159 success=True
160 160 failure=True
161 161
162 162 def __init__(self, dependencies=[], all=True, success=True, failure=False):
163 163 if isinstance(dependencies, dict):
164 164 # load from dict
165 165 all = dependencies.get('all', True)
166 166 success = dependencies.get('success', success)
167 167 failure = dependencies.get('failure', failure)
168 168 dependencies = dependencies.get('dependencies', [])
169 169 ids = []
170 170
171 171 # extract ids from various sources:
172 172 if isinstance(dependencies, string_types + (AsyncResult,)):
173 173 dependencies = [dependencies]
174 174 for d in dependencies:
175 175 if isinstance(d, string_types):
176 176 ids.append(d)
177 177 elif isinstance(d, AsyncResult):
178 178 ids.extend(d.msg_ids)
179 179 else:
180 180 raise TypeError("invalid dependency type: %r"%type(d))
181 181
182 182 set.__init__(self, ids)
183 183 self.all = all
184 184 if not (success or failure):
185 185 raise ValueError("Must depend on at least one of successes or failures!")
186 186 self.success=success
187 187 self.failure = failure
188 188
189 189 def check(self, completed, failed=None):
190 190 """check whether our dependencies have been met."""
191 191 if len(self) == 0:
192 192 return True
193 193 against = set()
194 194 if self.success:
195 195 against = completed
196 196 if failed is not None and self.failure:
197 197 against = against.union(failed)
198 198 if self.all:
199 199 return self.issubset(against)
200 200 else:
201 201 return not self.isdisjoint(against)
202 202
203 203 def unreachable(self, completed, failed=None):
204 204 """return whether this dependency has become impossible."""
205 205 if len(self) == 0:
206 206 return False
207 207 against = set()
208 208 if not self.success:
209 209 against = completed
210 210 if failed is not None and not self.failure:
211 211 against = against.union(failed)
212 212 if self.all:
213 213 return not self.isdisjoint(against)
214 214 else:
215 215 return self.issubset(against)
216 216
217 217
218 218 def as_dict(self):
219 219 """Represent this dependency as a dict. For json compatibility."""
220 220 return dict(
221 221 dependencies=list(self),
222 222 all=self.all,
223 223 success=self.success,
224 224 failure=self.failure
225 225 )
226 226
227 227
228 228 __all__ = ['depend', 'require', 'dependent', 'Dependency']
229 229
General Comments 0
You need to be logged in to leave comments. Login now