Show More
@@ -4,56 +4,33 b'' | |||||
4 |
|
4 | |||
5 | Scattering consists of partitioning a sequence and sending the various |
|
5 | Scattering consists of partitioning a sequence and sending the various | |
6 | pieces to individual nodes in a cluster. |
|
6 | pieces to individual nodes in a cluster. | |
7 |
|
||||
8 |
|
||||
9 | Authors: |
|
|||
10 |
|
||||
11 | * Brian Granger |
|
|||
12 | * MinRK |
|
|||
13 |
|
||||
14 | """ |
|
7 | """ | |
15 |
|
8 | |||
16 | #------------------------------------------------------------------------------- |
|
9 | # Copyright (c) IPython Development Team. | |
17 | # Copyright (C) 2008-2011 The IPython Development Team |
|
10 | # Distributed under the terms of the Modified BSD License. | |
18 | # |
|
|||
19 | # Distributed under the terms of the BSD License. The full license is in |
|
|||
20 | # the file COPYING, distributed as part of this software. |
|
|||
21 | #------------------------------------------------------------------------------- |
|
|||
22 |
|
||||
23 | #------------------------------------------------------------------------------- |
|
|||
24 | # Imports |
|
|||
25 | #------------------------------------------------------------------------------- |
|
|||
26 |
|
11 | |||
27 | from __future__ import division |
|
12 | from __future__ import division | |
28 |
|
13 | |||
|
14 | import sys | |||
29 | from itertools import islice |
|
15 | from itertools import islice | |
30 |
|
16 | |||
31 | from IPython.utils.data import flatten as utils_flatten |
|
17 | from IPython.utils.data import flatten as utils_flatten | |
32 |
|
18 | |||
33 | #------------------------------------------------------------------------------- |
|
|||
34 | # Figure out which array packages are present and their array types |
|
|||
35 | #------------------------------------------------------------------------------- |
|
|||
36 |
|
19 | |||
37 | arrayModules = [] |
|
20 | numpy = None | |
38 | try: |
|
21 | ||
39 | import Numeric |
|
22 | def is_array(obj): | |
40 | except ImportError: |
|
23 | """Is an object a numpy array? | |
41 |
|
|
24 | ||
42 | else: |
|
25 | Avoids importing numpy until it is requested | |
43 | arrayModules.append({'module':Numeric, 'type':Numeric.arraytype}) |
|
26 | """ | |
44 | try: |
|
27 | global numpy | |
|
28 | if 'numpy' not in sys.modules: | |||
|
29 | return False | |||
|
30 | ||||
|
31 | if numpy is None: | |||
45 | import numpy |
|
32 | import numpy | |
46 | except ImportError: |
|
33 | return isinstance(obj, numpy.ndarray) | |
47 | pass |
|
|||
48 | else: |
|
|||
49 | arrayModules.append({'module':numpy, 'type':numpy.ndarray}) |
|
|||
50 | try: |
|
|||
51 | import numarray |
|
|||
52 | except ImportError: |
|
|||
53 | pass |
|
|||
54 | else: |
|
|||
55 | arrayModules.append({'module':numarray, |
|
|||
56 | 'type':numarray.numarraycore.NumArray}) |
|
|||
57 |
|
34 | |||
58 | class Map(object): |
|
35 | class Map(object): | |
59 | """A class for partitioning a sequence using a map.""" |
|
36 | """A class for partitioning a sequence using a map.""" | |
@@ -94,10 +71,8 b' class Map(object):' | |||||
94 | def concatenate(self, listOfPartitions): |
|
71 | def concatenate(self, listOfPartitions): | |
95 | testObject = listOfPartitions[0] |
|
72 | testObject = listOfPartitions[0] | |
96 | # First see if we have a known array type |
|
73 | # First see if we have a known array type | |
97 |
f |
|
74 | if is_array(testObject): | |
98 | #print m |
|
75 | return numpy.concatenate(listOfPartitions) | |
99 | if isinstance(testObject, m['type']): |
|
|||
100 | return m['module'].concatenate(listOfPartitions) |
|
|||
101 | # Next try for Python sequence types |
|
76 | # Next try for Python sequence types | |
102 | if isinstance(testObject, (list, tuple)): |
|
77 | if isinstance(testObject, (list, tuple)): | |
103 | return utils_flatten(listOfPartitions) |
|
78 | return utils_flatten(listOfPartitions) | |
@@ -117,19 +92,17 b' class RoundRobinMap(Map):' | |||||
117 | def joinPartitions(self, listOfPartitions): |
|
92 | def joinPartitions(self, listOfPartitions): | |
118 | testObject = listOfPartitions[0] |
|
93 | testObject = listOfPartitions[0] | |
119 | # First see if we have a known array type |
|
94 | # First see if we have a known array type | |
120 |
f |
|
95 | if is_array(testObject): | |
121 | #print m |
|
96 | return self.flatten_array(listOfPartitions) | |
122 | if isinstance(testObject, m['type']): |
|
|||
123 | return self.flatten_array(m['type'], listOfPartitions) |
|
|||
124 | if isinstance(testObject, (list, tuple)): |
|
97 | if isinstance(testObject, (list, tuple)): | |
125 | return self.flatten_list(listOfPartitions) |
|
98 | return self.flatten_list(listOfPartitions) | |
126 | return listOfPartitions |
|
99 | return listOfPartitions | |
127 |
|
100 | |||
128 |
def flatten_array(self, |
|
101 | def flatten_array(self, listOfPartitions): | |
129 | test = listOfPartitions[0] |
|
102 | test = listOfPartitions[0] | |
130 | shape = list(test.shape) |
|
103 | shape = list(test.shape) | |
131 | shape[0] = sum([ p.shape[0] for p in listOfPartitions]) |
|
104 | shape[0] = sum([ p.shape[0] for p in listOfPartitions]) | |
132 |
A = |
|
105 | A = numpy.ndarray(shape) | |
133 | N = shape[0] |
|
106 | N = shape[0] | |
134 | q = len(listOfPartitions) |
|
107 | q = len(listOfPartitions) | |
135 | for p,part in enumerate(listOfPartitions): |
|
108 | for p,part in enumerate(listOfPartitions): | |
@@ -141,22 +114,12 b' class RoundRobinMap(Map):' | |||||
141 | for i in range(len(listOfPartitions[0])): |
|
114 | for i in range(len(listOfPartitions[0])): | |
142 | flat.extend([ part[i] for part in listOfPartitions if len(part) > i ]) |
|
115 | flat.extend([ part[i] for part in listOfPartitions if len(part) > i ]) | |
143 | return flat |
|
116 | return flat | |
144 | #lengths = [len(x) for x in listOfPartitions] |
|
|||
145 | #maxPartitionLength = len(listOfPartitions[0]) |
|
|||
146 | #numberOfPartitions = len(listOfPartitions) |
|
|||
147 | #concat = self.concatenate(listOfPartitions) |
|
|||
148 | #totalLength = len(concat) |
|
|||
149 | #result = [] |
|
|||
150 | #for i in range(maxPartitionLength): |
|
|||
151 | # result.append(concat[i:totalLength:maxPartitionLength]) |
|
|||
152 | # return self.concatenate(listOfPartitions) |
|
|||
153 |
|
117 | |||
154 | def mappable(obj): |
|
118 | def mappable(obj): | |
155 | """return whether an object is mappable or not.""" |
|
119 | """return whether an object is mappable or not.""" | |
156 | if isinstance(obj, (tuple,list)): |
|
120 | if isinstance(obj, (tuple,list)): | |
157 | return True |
|
121 | return True | |
158 | for m in arrayModules: |
|
122 | if is_array(obj): | |
159 | if isinstance(obj,m['type']): |
|
|||
160 |
|
|
123 | return True | |
161 | return False |
|
124 | return False | |
162 |
|
125 |
General Comments 0
You need to be logged in to leave comments.
Login now