##// END OF EJS Templates
don't use common names in require decorators...
MinRK -
Show More
@@ -1,229 +1,229 b''
1 """Dependency utilities
1 """Dependency utilities
2
2
3 Authors:
3 Authors:
4
4
5 * Min RK
5 * Min RK
6 """
6 """
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2013 The IPython Development Team
8 # Copyright (C) 2013 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 from types import ModuleType
14 from types import ModuleType
15
15
16 from IPython.parallel.client.asyncresult import AsyncResult
16 from IPython.parallel.client.asyncresult import AsyncResult
17 from IPython.parallel.error import UnmetDependency
17 from IPython.parallel.error import UnmetDependency
18 from IPython.parallel.util import interactive
18 from IPython.parallel.util import interactive
19 from IPython.utils import py3compat
19 from IPython.utils import py3compat
20 from IPython.utils.py3compat import string_types
20 from IPython.utils.py3compat import string_types
21 from IPython.utils.pickleutil import can, uncan
21 from IPython.utils.pickleutil import can, uncan
22
22
23 class depend(object):
23 class depend(object):
24 """Dependency decorator, for use with tasks.
24 """Dependency decorator, for use with tasks.
25
25
26 `@depend` lets you define a function for engine dependencies
26 `@depend` lets you define a function for engine dependencies
27 just like you use `apply` for tasks.
27 just like you use `apply` for tasks.
28
28
29
29
30 Examples
30 Examples
31 --------
31 --------
32 ::
32 ::
33
33
34 @depend(df, a,b, c=5)
34 @depend(df, a,b, c=5)
35 def f(m,n,p)
35 def f(m,n,p)
36
36
37 view.apply(f, 1,2,3)
37 view.apply(f, 1,2,3)
38
38
39 will call df(a,b,c=5) on the engine, and if it returns False or
39 will call df(a,b,c=5) on the engine, and if it returns False or
40 raises an UnmetDependency error, then the task will not be run
40 raises an UnmetDependency error, then the task will not be run
41 and another engine will be tried.
41 and another engine will be tried.
42 """
42 """
43 def __init__(self, f, *args, **kwargs):
43 def __init__(self, _wrapped_f, *args, **kwargs):
44 self.f = f
44 self.f = _wrapped_f
45 self.args = args
45 self.args = args
46 self.kwargs = kwargs
46 self.kwargs = kwargs
47
47
48 def __call__(self, f):
48 def __call__(self, f):
49 return dependent(f, self.f, *self.args, **self.kwargs)
49 return dependent(f, self.f, *self.args, **self.kwargs)
50
50
51 class dependent(object):
51 class dependent(object):
52 """A function that depends on another function.
52 """A function that depends on another function.
53 This is an object to prevent the closure used
53 This is an object to prevent the closure used
54 in traditional decorators, which are not picklable.
54 in traditional decorators, which are not picklable.
55 """
55 """
56
56
57 def __init__(self, f, df, *dargs, **dkwargs):
57 def __init__(self, _wrapped_f, _wrapped_df, *dargs, **dkwargs):
58 self.f = f
58 self.f = _wrapped_f
59 name = getattr(f, '__name__', 'f')
59 name = getattr(f, '__name__', 'f')
60 if py3compat.PY3:
60 if py3compat.PY3:
61 self.__name__ = name
61 self.__name__ = name
62 else:
62 else:
63 self.func_name = name
63 self.func_name = name
64 self.df = df
64 self.df = _wrapped_df
65 self.dargs = dargs
65 self.dargs = dargs
66 self.dkwargs = dkwargs
66 self.dkwargs = dkwargs
67
67
68 def check_dependency(self):
68 def check_dependency(self):
69 if self.df(*self.dargs, **self.dkwargs) is False:
69 if self.df(*self.dargs, **self.dkwargs) is False:
70 raise UnmetDependency()
70 raise UnmetDependency()
71
71
72 def __call__(self, *args, **kwargs):
72 def __call__(self, *args, **kwargs):
73 return self.f(*args, **kwargs)
73 return self.f(*args, **kwargs)
74
74
75 if not py3compat.PY3:
75 if not py3compat.PY3:
76 @property
76 @property
77 def __name__(self):
77 def __name__(self):
78 return self.func_name
78 return self.func_name
79
79
80 @interactive
80 @interactive
81 def _require(*modules, **mapping):
81 def _require(*modules, **mapping):
82 """Helper for @require decorator."""
82 """Helper for @require decorator."""
83 from IPython.parallel.error import UnmetDependency
83 from IPython.parallel.error import UnmetDependency
84 from IPython.utils.pickleutil import uncan
84 from IPython.utils.pickleutil import uncan
85 user_ns = globals()
85 user_ns = globals()
86 for name in modules:
86 for name in modules:
87 try:
87 try:
88 exec('import %s' % name, user_ns)
88 exec('import %s' % name, user_ns)
89 except ImportError:
89 except ImportError:
90 raise UnmetDependency(name)
90 raise UnmetDependency(name)
91
91
92 for name, cobj in mapping.items():
92 for name, cobj in mapping.items():
93 user_ns[name] = uncan(cobj, user_ns)
93 user_ns[name] = uncan(cobj, user_ns)
94 return True
94 return True
95
95
96 def require(*objects, **mapping):
96 def require(*objects, **mapping):
97 """Simple decorator for requiring local objects and modules to be available
97 """Simple decorator for requiring local objects and modules to be available
98 when the decorated function is called on the engine.
98 when the decorated function is called on the engine.
99
99
100 Modules specified by name or passed directly will be imported
100 Modules specified by name or passed directly will be imported
101 prior to calling the decorated function.
101 prior to calling the decorated function.
102
102
103 Objects other than modules will be pushed as a part of the task.
103 Objects other than modules will be pushed as a part of the task.
104 Functions can be passed positionally,
104 Functions can be passed positionally,
105 and will be pushed to the engine with their __name__.
105 and will be pushed to the engine with their __name__.
106 Other objects can be passed by keyword arg.
106 Other objects can be passed by keyword arg.
107
107
108 Examples::
108 Examples::
109
109
110 In [1]: @require('numpy')
110 In [1]: @require('numpy')
111 ...: def norm(a):
111 ...: def norm(a):
112 ...: return numpy.linalg.norm(a,2)
112 ...: return numpy.linalg.norm(a,2)
113
113
114 In [2]: foo = lambda x: x*x
114 In [2]: foo = lambda x: x*x
115 In [3]: @require(foo)
115 In [3]: @require(foo)
116 ...: def bar(a):
116 ...: def bar(a):
117 ...: return foo(1-a)
117 ...: return foo(1-a)
118 """
118 """
119 names = []
119 names = []
120 for obj in objects:
120 for obj in objects:
121 if isinstance(obj, ModuleType):
121 if isinstance(obj, ModuleType):
122 obj = obj.__name__
122 obj = obj.__name__
123
123
124 if isinstance(obj, string_types):
124 if isinstance(obj, string_types):
125 names.append(obj)
125 names.append(obj)
126 elif hasattr(obj, '__name__'):
126 elif hasattr(obj, '__name__'):
127 mapping[obj.__name__] = obj
127 mapping[obj.__name__] = obj
128 else:
128 else:
129 raise TypeError("Objects other than modules and functions "
129 raise TypeError("Objects other than modules and functions "
130 "must be passed by kwarg, but got: %s" % type(obj)
130 "must be passed by kwarg, but got: %s" % type(obj)
131 )
131 )
132
132
133 for name, obj in mapping.items():
133 for name, obj in mapping.items():
134 mapping[name] = can(obj)
134 mapping[name] = can(obj)
135 return depend(_require, *names, **mapping)
135 return depend(_require, *names, **mapping)
136
136
137 class Dependency(set):
137 class Dependency(set):
138 """An object for representing a set of msg_id dependencies.
138 """An object for representing a set of msg_id dependencies.
139
139
140 Subclassed from set().
140 Subclassed from set().
141
141
142 Parameters
142 Parameters
143 ----------
143 ----------
144 dependencies: list/set of msg_ids or AsyncResult objects or output of Dependency.as_dict()
144 dependencies: list/set of msg_ids or AsyncResult objects or output of Dependency.as_dict()
145 The msg_ids to depend on
145 The msg_ids to depend on
146 all : bool [default True]
146 all : bool [default True]
147 Whether the dependency should be considered met when *all* depending tasks have completed
147 Whether the dependency should be considered met when *all* depending tasks have completed
148 or only when *any* have been completed.
148 or only when *any* have been completed.
149 success : bool [default True]
149 success : bool [default True]
150 Whether to consider successes as fulfilling dependencies.
150 Whether to consider successes as fulfilling dependencies.
151 failure : bool [default False]
151 failure : bool [default False]
152 Whether to consider failures as fulfilling dependencies.
152 Whether to consider failures as fulfilling dependencies.
153
153
154 If `all=success=True` and `failure=False`, then the task will fail with an ImpossibleDependency
154 If `all=success=True` and `failure=False`, then the task will fail with an ImpossibleDependency
155 as soon as the first depended-upon task fails.
155 as soon as the first depended-upon task fails.
156 """
156 """
157
157
158 all=True
158 all=True
159 success=True
159 success=True
160 failure=True
160 failure=True
161
161
162 def __init__(self, dependencies=[], all=True, success=True, failure=False):
162 def __init__(self, dependencies=[], all=True, success=True, failure=False):
163 if isinstance(dependencies, dict):
163 if isinstance(dependencies, dict):
164 # load from dict
164 # load from dict
165 all = dependencies.get('all', True)
165 all = dependencies.get('all', True)
166 success = dependencies.get('success', success)
166 success = dependencies.get('success', success)
167 failure = dependencies.get('failure', failure)
167 failure = dependencies.get('failure', failure)
168 dependencies = dependencies.get('dependencies', [])
168 dependencies = dependencies.get('dependencies', [])
169 ids = []
169 ids = []
170
170
171 # extract ids from various sources:
171 # extract ids from various sources:
172 if isinstance(dependencies, string_types + (AsyncResult,)):
172 if isinstance(dependencies, string_types + (AsyncResult,)):
173 dependencies = [dependencies]
173 dependencies = [dependencies]
174 for d in dependencies:
174 for d in dependencies:
175 if isinstance(d, string_types):
175 if isinstance(d, string_types):
176 ids.append(d)
176 ids.append(d)
177 elif isinstance(d, AsyncResult):
177 elif isinstance(d, AsyncResult):
178 ids.extend(d.msg_ids)
178 ids.extend(d.msg_ids)
179 else:
179 else:
180 raise TypeError("invalid dependency type: %r"%type(d))
180 raise TypeError("invalid dependency type: %r"%type(d))
181
181
182 set.__init__(self, ids)
182 set.__init__(self, ids)
183 self.all = all
183 self.all = all
184 if not (success or failure):
184 if not (success or failure):
185 raise ValueError("Must depend on at least one of successes or failures!")
185 raise ValueError("Must depend on at least one of successes or failures!")
186 self.success=success
186 self.success=success
187 self.failure = failure
187 self.failure = failure
188
188
189 def check(self, completed, failed=None):
189 def check(self, completed, failed=None):
190 """check whether our dependencies have been met."""
190 """check whether our dependencies have been met."""
191 if len(self) == 0:
191 if len(self) == 0:
192 return True
192 return True
193 against = set()
193 against = set()
194 if self.success:
194 if self.success:
195 against = completed
195 against = completed
196 if failed is not None and self.failure:
196 if failed is not None and self.failure:
197 against = against.union(failed)
197 against = against.union(failed)
198 if self.all:
198 if self.all:
199 return self.issubset(against)
199 return self.issubset(against)
200 else:
200 else:
201 return not self.isdisjoint(against)
201 return not self.isdisjoint(against)
202
202
203 def unreachable(self, completed, failed=None):
203 def unreachable(self, completed, failed=None):
204 """return whether this dependency has become impossible."""
204 """return whether this dependency has become impossible."""
205 if len(self) == 0:
205 if len(self) == 0:
206 return False
206 return False
207 against = set()
207 against = set()
208 if not self.success:
208 if not self.success:
209 against = completed
209 against = completed
210 if failed is not None and not self.failure:
210 if failed is not None and not self.failure:
211 against = against.union(failed)
211 against = against.union(failed)
212 if self.all:
212 if self.all:
213 return not self.isdisjoint(against)
213 return not self.isdisjoint(against)
214 else:
214 else:
215 return self.issubset(against)
215 return self.issubset(against)
216
216
217
217
218 def as_dict(self):
218 def as_dict(self):
219 """Represent this dependency as a dict. For json compatibility."""
219 """Represent this dependency as a dict. For json compatibility."""
220 return dict(
220 return dict(
221 dependencies=list(self),
221 dependencies=list(self),
222 all=self.all,
222 all=self.all,
223 success=self.success,
223 success=self.success,
224 failure=self.failure
224 failure=self.failure
225 )
225 )
226
226
227
227
228 __all__ = ['depend', 'require', 'dependent', 'Dependency']
228 __all__ = ['depend', 'require', 'dependent', 'Dependency']
229
229
General Comments 0
You need to be logged in to leave comments. Login now