##// END OF EJS Templates
support iterators in view.map...
MinRK -
Show More
@@ -27,6 +27,7 b' Authors:'
27 from __future__ import division
27 from __future__ import division
28
28
29 import types
29 import types
30 from itertools import islice
30
31
31 from IPython.utils.data import flatten as utils_flatten
32 from IPython.utils.data import flatten as utils_flatten
32
33
@@ -77,9 +78,14 b' class Map:'
77 else:
78 else:
78 lo.append(n*basesize + remainder)
79 lo.append(n*basesize + remainder)
79 hi.append(lo[-1] + basesize)
80 hi.append(lo[-1] + basesize)
80
81
81
82 result = seq[lo[p]:hi[p]]
82 try:
83 result = seq[lo[p]:hi[p]]
84 except TypeError:
85 # some objects (iterators) can't be sliced,
86 # use islice:
87 result = list(islice(seq, lo[p], hi[p]))
88
83 return result
89 return result
84
90
85 def joinPartitions(self, listOfPartitions):
91 def joinPartitions(self, listOfPartitions):
@@ -95,6 +95,17 b' class TestLoadBalancedView(ClusterTestCase):'
95 # Ensure that results came in order
95 # Ensure that results came in order
96 self.assertEquals(astheycame, reference)
96 self.assertEquals(astheycame, reference)
97 self.assertEquals(amr.result, reference)
97 self.assertEquals(amr.result, reference)
98
99 def test_map_iterable(self):
100 """test map on iterables (balanced)"""
101 view = self.view
102 # 101 is prime, so it won't be evenly distributed
103 arr = range(101)
104 # so that it will be an iterator, even in Python 3
105 it = iter(arr)
106 r = view.map_sync(lambda x:x, arr)
107 self.assertEquals(r, list(arr))
108
98
109
99 def test_abort(self):
110 def test_abort(self):
100 view = self.view
111 view = self.view
@@ -240,6 +240,16 b' class TestView(ClusterTestCase):'
240 r = view.map_sync(f, data)
240 r = view.map_sync(f, data)
241 self.assertEquals(r, map(f, data))
241 self.assertEquals(r, map(f, data))
242
242
243 def test_map_iterable(self):
244 """test map on iterables (direct)"""
245 view = self.client[:]
246 # 101 is prime, so it won't be evenly distributed
247 arr = range(101)
248 # ensure it will be an iterator, even in Python 3
249 it = iter(arr)
250 r = view.map_sync(lambda x:x, arr)
251 self.assertEquals(r, list(arr))
252
243 def test_scatterGatherNonblocking(self):
253 def test_scatterGatherNonblocking(self):
244 data = range(16)
254 data = range(16)
245 view = self.client[:]
255 view = self.client[:]
@@ -446,6 +456,5 b' class TestView(ClusterTestCase):'
446 self.fail(e.evalue)
456 self.fail(e.evalue)
447 else:
457 else:
448 raise e
458 raise e
449
459
450
451
460
General Comments 0
You need to be logged in to leave comments. Login now