From 9f98b808e8139b8241e65c3fd41f466be137cdfd Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Wed, 28 May 2014 12:43:05 -0500 Subject: [PATCH 1/7] Add failing test. --- distarray/dist/tests/test_distarray.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index 0cc7c261..e9ae282c 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -158,6 +158,13 @@ def test_slice_a_slice_block_dist_1(self): s2 = s1[-2:] assert_array_equal(s2.toarray(), expected[3:5]) + def test_slice_block_dist_1d_with_step(self): + size = 10 + step = 2 + expected = numpy.random.randint(10, size=size) + darr = self.dac.fromarray(expected) + assert_array_equal(darr[::2], expected[::2]) + def test_partial_slice_block_dist_2d(self): shape = (10, 20) expected = numpy.random.randint(10, size=shape) From 678d76dea804f7ee6c6cf81542125085d0b68232 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Wed, 28 May 2014 16:10:49 -0500 Subject: [PATCH 2/7] Generalize tuple_intersection. --- distarray/metadata_utils.py | 36 ++++++++++++++---- distarray/tests/test_metadata_utils.py | 52 ++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/distarray/metadata_utils.py b/distarray/metadata_utils.py index 713ea0e9..7a6c3ff4 100644 --- a/distarray/metadata_utils.py +++ b/distarray/metadata_utils.py @@ -4,6 +4,8 @@ # Distributed under the terms of the BSD License. See COPYING.rst. # --------------------------------------------------------------------------- +from __future__ import division + import operator from itertools import product from functools import reduce @@ -228,20 +230,39 @@ def _check_bounds(index, size): raise IndexError("Index %r out of bounds" % index) -def tuple_intersection(t1, t2): - """Compute intersection of two (start, stop) tuples. +def tuple_intersection(t0, t1): + """Compute intersection of a (start, stop, step) and a (start, stop) tuple. + + Assumes all values are positive. Parameters ---------- - t1, t2 : 2-tuples + t0: 2-tuple or 3-tuple + Tuple of (start, stop, [step]) representing an index range + t1: 2-tuple + Tuple of (start, stop) representing an index range Returns ------- - 2-tuple or None + 3-tuple or None + A tightly bounded interval. """ - stop = min(t1[1], t2[1]) - start = max(t1[0], t2[0]) - return (start, stop) if stop - start > 0 else None + if len(t0) == 2: + # default step is 1 + t0 = t0 + (1,) + + start0, stop0, step0 = t0 + start1, stop1 = t1 + n = int(numpy.ceil((start1 - start0) / step0)) + start2 = start0 + n*step0 + + max_stop = min(t0[1], t1[1]) + if (max_stop - start2) % step0 == 0: + n = ((max_stop - start2) // step0) - 1 + else: + n = (max_stop - start2) // step0 + stop2 = (start2 + n*step0) + 1 + return (start2, stop2, step0) if stop2 > start2 else None def positivify(index, size): @@ -324,7 +345,6 @@ def replace_ellipsis(idx): return idx sanitized = tuple(replace_ellipsis(i) for i in sanitized) - if ndim is not None: diff = ndim - len(sanitized) if diff < 0: diff --git a/distarray/tests/test_metadata_utils.py b/distarray/tests/test_metadata_utils.py index be5ea0d3..6cb4b67c 100644 --- a/distarray/tests/test_metadata_utils.py +++ b/distarray/tests/test_metadata_utils.py @@ -138,6 +138,58 @@ def test_multiple_ellipsis(self): ndim=ndim) self.assertEqual(sanitized, (slice(None),) * 4 + (10, slice(None))) + def test_step(self): + # currently doesn't touch step + indices = (slice(None, None, 2), slice(None, 8, 4)) + tag, sanitized = metadata_utils.sanitize_indices(indices) + self.assertEqual(tag, 'view') + self.assertEqual(sanitized, indices) + + +class TestTupleIntersection(unittest.TestCase): + + def test_no_step_full_enclosure(self): + t0 = (0, 60) + t1 = (15, 30) + result = metadata_utils.tuple_intersection(t0, t1) + self.assertEqual(result, (15, 30, 1)) + + def test_no_step_partial_overlap(self): + t0 = (0, 60) + t1 = (15, 90) + result = metadata_utils.tuple_intersection(t0, t1) + self.assertEqual(result, (15, 60, 1)) + + def test_no_step_no_overlap(self): + t0 = (0, 60) + t1 = (80, 130) + result = metadata_utils.tuple_intersection(t0, t1) + self.assertEqual(result, None) + + def test_with_step_1(self): + t0 = (0, 60, 1) + t1 = (15, 30) + result = metadata_utils.tuple_intersection(t0, t1) + self.assertEqual(result, (15, 30, 1)) + + def test_with_step_2(self): + t0 = (0, 60, 2) + t1 = (15, 30) + result = metadata_utils.tuple_intersection(t0, t1) + self.assertEqual(result, (16, 29, 2)) + + def test_with_step_3(self): + t0 = (0, 59, 2) + t1 = (15, 90) + result = metadata_utils.tuple_intersection(t0, t1) + self.assertEqual(result, (16, 59, 2)) + + def test_big_step(self): + t0 = (0, 59, 1000) + t1 = (15, 90) + result = metadata_utils.tuple_intersection(t0, t1) + self.assertEqual(result, None) + if __name__ == '__main__': unittest.main(verbosity=2) From 53ca6122ed7cc3670166633589f312ce960c7841 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Thu, 29 May 2014 18:26:43 -0500 Subject: [PATCH 3/7] Fix tuple_intersection for start0 >= start1. Also, allow passing in a 2-tuple for t0. --- distarray/metadata_utils.py | 11 ++++--- distarray/tests/test_metadata_utils.py | 43 ++++++++++++++++++++------ 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/distarray/metadata_utils.py b/distarray/metadata_utils.py index df096a37..a7f59190 100644 --- a/distarray/metadata_utils.py +++ b/distarray/metadata_utils.py @@ -247,14 +247,17 @@ def tuple_intersection(t0, t1): 3-tuple or None A tightly bounded interval. """ - if len(t0) == 2: + if len(t0) == 2 or t0[2] is None: # default step is 1 - t0 = t0 + (1,) + t0 = (t0[0], t0[1], 1) start0, stop0, step0 = t0 start1, stop1 = t1 - n = int(numpy.ceil((start1 - start0) / step0)) - start2 = start0 + n*step0 + if start0 < start1: + n = int(numpy.ceil((start1 - start0) / step0)) + start2 = start0 + n*step0 + else: + start2 = start0 max_stop = min(t0[1], t1[1]) if (max_stop - start2) % step0 == 0: diff --git a/distarray/tests/test_metadata_utils.py b/distarray/tests/test_metadata_utils.py index 6cb4b67c..577a1c30 100644 --- a/distarray/tests/test_metadata_utils.py +++ b/distarray/tests/test_metadata_utils.py @@ -148,47 +148,70 @@ def test_step(self): class TestTupleIntersection(unittest.TestCase): + def check_intersection_and_reverse(self, t0, t1, expected): + result = metadata_utils.tuple_intersection(t0, t1) + self.assertEqual(result, expected) + result = metadata_utils.tuple_intersection(t1, t0) + self.assertEqual(result, expected) + def test_no_step_full_enclosure(self): t0 = (0, 60) t1 = (15, 30) - result = metadata_utils.tuple_intersection(t0, t1) - self.assertEqual(result, (15, 30, 1)) + expected = (15, 30, 1) + self.check_intersection_and_reverse(t0, t1, expected) def test_no_step_partial_overlap(self): t0 = (0, 60) t1 = (15, 90) - result = metadata_utils.tuple_intersection(t0, t1) - self.assertEqual(result, (15, 60, 1)) + expected = (15, 60, 1) + self.check_intersection_and_reverse(t0, t1, expected) def test_no_step_no_overlap(self): t0 = (0, 60) t1 = (80, 130) - result = metadata_utils.tuple_intersection(t0, t1) - self.assertEqual(result, None) + expected = None + self.check_intersection_and_reverse(t0, t1, expected) + + def test_no_step_partial_overlap_0(self): + t0 = (0, 60) + t1 = (15, 90) + expected = (15, 60, 1) + self.check_intersection_and_reverse(t0, t1, expected) + + def test_no_step_partial_overlap_1(self): + # regression test + t0 = (0, 4) + t1 = (3, 7) + expected = (3, 4, 1) + self.check_intersection_and_reverse(t0, t1, expected) def test_with_step_1(self): t0 = (0, 60, 1) t1 = (15, 30) + expected = (15, 30, 1) result = metadata_utils.tuple_intersection(t0, t1) - self.assertEqual(result, (15, 30, 1)) + self.assertSequenceEqual(result, expected) def test_with_step_2(self): t0 = (0, 60, 2) t1 = (15, 30) + expected = (16, 29, 2) result = metadata_utils.tuple_intersection(t0, t1) - self.assertEqual(result, (16, 29, 2)) + self.assertSequenceEqual(result, expected) def test_with_step_3(self): t0 = (0, 59, 2) t1 = (15, 90) + expected = (16, 59, 2) result = metadata_utils.tuple_intersection(t0, t1) - self.assertEqual(result, (16, 59, 2)) + self.assertSequenceEqual(result, expected) def test_big_step(self): t0 = (0, 59, 1000) t1 = (15, 90) + expected = None result = metadata_utils.tuple_intersection(t0, t1) - self.assertEqual(result, None) + self.assertEqual(result, expected) if __name__ == '__main__': From 913f4a06891a113a7012f320b0022ae8736eab63 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Thu, 29 May 2014 18:54:09 -0500 Subject: [PATCH 4/7] Slicing with steps. --- distarray/dist/maps.py | 49 +++++++++++++++----------- distarray/dist/tests/test_distarray.py | 2 +- distarray/dist/tests/test_maps.py | 26 ++++++++++++++ distarray/local/maps.py | 12 +++++-- 4 files changed, 66 insertions(+), 23 deletions(-) diff --git a/distarray/dist/maps.py b/distarray/dist/maps.py index eb9e5b48..c83a1fad 100644 --- a/distarray/dist/maps.py +++ b/distarray/dist/maps.py @@ -21,7 +21,7 @@ `UnstructuredMap`. """ -from __future__ import absolute_import +from __future__ import division, absolute_import import operator from itertools import product @@ -200,7 +200,13 @@ def owners(self, idx): if isinstance(idx, Integral): return [0] if 0 <= idx < self.size else [] elif isinstance(idx, slice): - return [0] # slicing doesn't complain about out-of-bounds indices + start = idx.start if idx.start is not None else 0 + stop = idx.stop if idx.stop is not None else self.size + step = idx.step if idx.step is not None else 1 + if tuple_intersection((start, stop, step), (0, self.size)): + return [0] + else: + return [] else: raise TypeError("Index must be Integral or slice.") @@ -216,14 +222,16 @@ def slice(self, idx): """Make a new Map from a slice.""" start = idx.start if idx.start is not None else 0 stop = idx.stop if idx.stop is not None else self.size - intersection = tuple_intersection((0, self.size), (start, stop)) - if intersection: - intersection_size = intersection[1] - intersection[0] + step = idx.step if idx.step is not None else 1 + isection = tuple_intersection((start, stop, step), (0, self.size)) + if isection: + step = idx.step if idx.step is not None else 1 + isection_size = int(np.ceil((isection[1] - isection[0]) // step)) else: - intersection_size = 0 + isection_size = 0 return {'dist_type': self.dist, - 'size': intersection_size} + 'size': isection_size} class BlockMap(MapBase): @@ -282,15 +290,13 @@ def owners(self, idx): coords.append(coord) return coords elif isinstance(idx, slice): - if idx.step not in {None, 1}: - msg = "Slicing only implemented for step=1" - raise NotImplementedError(msg) + start = idx.start if idx.start is not None else 0 + stop = idx.stop if idx.stop is not None else self.size + step = idx.step if idx.step is not None else 1 for (coord, (lower, upper)) in enumerate(self.bounds): - slice_tuple = (idx.start if idx.start is not None else 0, - idx.stop if idx.stop is not None else self.size) - if tuple_intersection((lower, upper), slice_tuple): + if tuple_intersection((start, stop, step), (lower, upper)): coords.append(coord) - return coords if coords != [] else [0] + return coords else: raise TypeError("Index must be Integral or slice.") @@ -319,14 +325,17 @@ def slice(self, idx): """Make a new Map from a slice.""" new_bounds = [0] start = idx.start if idx.start is not None else 0 + step = idx.step if idx.step is not None else 1 # iterate over the processes in this dimension for proc_start, proc_stop in self.bounds: stop = idx.stop if idx.stop is not None else proc_stop - intersection = tuple_intersection((proc_start, proc_stop), - (start, stop)) - if intersection: - size = intersection[1] - intersection[0] - new_bounds.append(size + new_bounds[-1]) + isection = tuple_intersection((start, stop, step), + (proc_start, proc_stop)) + if isection: + isection_size = int(np.ceil((isection[1] - (isection[0])) / step)) + new_bounds.append(isection_size + new_bounds[-1]) + if len(new_bounds) == [0]: + new_bounds = [] return {'dist_type': self.dist, 'bounds': new_bounds} @@ -429,7 +438,7 @@ def owners(self, idx): if isinstance(idx, Integral): return self._owners else: - msg = "Index for BlockCyclicMap must be an Integral." + msg = "Index for UnstructuredMap must be an Integral." raise NotImplementedError(msg) def get_dimdicts(self): diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index b3d0d7a6..0892a42b 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -163,7 +163,7 @@ def test_slice_block_dist_1d_with_step(self): step = 2 expected = numpy.random.randint(10, size=size) darr = self.dac.fromarray(expected) - assert_array_equal(darr[::2], expected[::2]) + assert_array_equal(darr[::step].toarray(), expected[::step]) def test_partial_slice_block_dist_2d(self): shape = (10, 20) diff --git a/distarray/dist/tests/test_maps.py b/distarray/dist/tests/test_maps.py index bab31f4d..e187ddc6 100644 --- a/distarray/dist/tests/test_maps.py +++ b/distarray/dist/tests/test_maps.py @@ -4,7 +4,10 @@ # Distributed under the terms of the BSD License. See COPYING.rst. # --------------------------------------------------------------------------- +from __future__ import division + import unittest +import numpy from random import randrange from distarray.externals.six.moves import range @@ -141,6 +144,29 @@ def test_from_full_slice_1d(self): self.assertSequenceEqual(d1.targets, d0.targets) self.assertSequenceEqual(d1.maps[0].bounds, d0.maps[0].bounds) + def test_from_full_slice_with_step_1d_0(self): + d0 = maps.Distribution.from_shape(context=self.ctx, shape=(15,)) + + s = (slice(None, None, 2),) + d1 = d0.slice(s) + + self.assertEqual(len(d0.maps), len(d1.maps)) + self.assertSequenceEqual(d1.dist, d0.dist) + self.assertSequenceEqual(d1.targets, d0.targets) + self.assertEqual(d1.maps[0].bounds[0][0], d0.maps[0].bounds[0][0]) + + def test_from_full_slice_with_step_1d_1(self): + d0 = maps.Distribution.from_shape(context=self.ctx, shape=(30,)) + step = 4 + + s = (slice(4, None, step),) + d1 = d0.slice(s) + + self.assertEqual(len(d0.maps), len(d1.maps)) + self.assertSequenceEqual(d1.dist, d0.dist) + self.assertSequenceEqual(d1.targets, d0.targets) + self.assertEqual(d1.maps[0].bounds[0][0], d0.maps[0].bounds[0][0]) + def test_from_full_slice_2d(self): d0 = maps.Distribution.from_shape(context=self.ctx, shape=(15, 20)) diff --git a/distarray/local/maps.py b/distarray/local/maps.py index b775d8d3..5cd66c35 100644 --- a/distarray/local/maps.py +++ b/distarray/local/maps.py @@ -213,11 +213,19 @@ def local_from_global(self, gidx): raise IndexError("Global index %s out of bounds" % gidx) return gidx - self.start elif isinstance(gidx, slice): + # we don't make the effort to compute the exact slice + # `__getitem__` doesn't care about overly-large slices, we just + # have to get the offset from the start correct based on the `step` start = gidx.start if gidx.start is not None else 0 stop = gidx.stop if gidx.stop is not None else self.global_size - new_start = max(start - self.start, 0) # prevent negative inds + step = gidx.step if gidx.step is not None else 1 + new_start = start - self.start + if new_start < 0: # don't allow negative starts + new_start += step * abs(new_start // step) + if new_start < 0: + new_start += step new_stop = stop - self.start - return slice(new_start, new_stop) + return slice(new_start, new_stop, gidx.step) else: raise TypeError("Index must be Integral or slice.") From 546acaca9dfb3222312ce43ccc3190b31d05884c Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Thu, 29 May 2014 19:54:29 -0500 Subject: [PATCH 5/7] Fix a bug. --- distarray/dist/maps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distarray/dist/maps.py b/distarray/dist/maps.py index c83a1fad..ae1fc0f6 100644 --- a/distarray/dist/maps.py +++ b/distarray/dist/maps.py @@ -226,7 +226,7 @@ def slice(self, idx): isection = tuple_intersection((start, stop, step), (0, self.size)) if isection: step = idx.step if idx.step is not None else 1 - isection_size = int(np.ceil((isection[1] - isection[0]) // step)) + isection_size = int(np.ceil((isection[1] - isection[0]) / step)) else: isection_size = 0 From 2b45d3fdbf0732ca4d9892fd566b2780ab978f68 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Thu, 29 May 2014 19:54:39 -0500 Subject: [PATCH 6/7] Add some setitem tests. --- distarray/dist/tests/test_distarray.py | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index 0892a42b..e1e6531d 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -245,6 +245,15 @@ def test_large_1d_slice(self): arr[slc] = new_data assert_array_equal(arr.toarray(), source) + def test_1d_slice_with_step(self): + source = numpy.random.randint(10, size=20) + new_data = numpy.random.randint(10, size=5) + slc = slice(7, 17, 2) + arr = self.dac.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + def test_2d_slice_0(self): # on process boundaries source = numpy.random.randint(10, size=(10, 20)) @@ -255,6 +264,15 @@ def test_2d_slice_0(self): arr[slc] = new_data assert_array_equal(arr.toarray(), source) + def test_2d_slice_with_step(self): + source = numpy.random.randint(10, size=(10, 20)) + new_data = numpy.random.randint(10, size=(2, 5)) + slc = (slice(5, 10, 3), slice(5, 15, 2)) + arr = self.dac.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + def test_2d_slice_1(self): # off process boundaries source = numpy.random.randint(10, size=(10, 20)) @@ -283,6 +301,15 @@ def test_full_3d_slice_ellipsis(self): arr[slc] = new_data assert_array_equal(arr.toarray(), source) + def test_3d_slice_ellipsis_with_step(self): + source = numpy.random.randint(10, size=(5, 4, 5)) + new_data = numpy.random.randint(10, size=(5, 2, 5)) + slc = (Ellipsis, slice(None, None, 2), Ellipsis) + arr = self.dac.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + def test_partial_indexing_0(self): source = numpy.random.randint(10, size=(3, 4, 5)) new_data = numpy.random.randint(10, size=(4, 5)) From 5e933e477953db868d5f372e2b84ed05993649d9 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Thu, 5 Jun 2014 12:42:09 -0500 Subject: [PATCH 7/7] Fix tests. --- distarray/dist/tests/test_distarray.py | 8 ++++---- distarray/dist/tests/test_maps.py | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index 69e94040..479f8798 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -150,7 +150,7 @@ def test_slice_block_dist_1d_with_step(self): size = 10 step = 2 expected = numpy.random.randint(10, size=size) - darr = self.dac.fromarray(expected) + darr = self.context.fromarray(expected) assert_array_equal(darr[::step].toarray(), expected[::step]) def test_partial_slice_block_dist_2d(self): @@ -229,7 +229,7 @@ def test_1d_slice_with_step(self): source = numpy.random.randint(10, size=20) new_data = numpy.random.randint(10, size=5) slc = slice(7, 17, 2) - arr = self.dac.fromarray(source) + arr = self.context.fromarray(source) source[slc] = new_data arr[slc] = new_data assert_array_equal(arr.toarray(), source) @@ -248,7 +248,7 @@ def test_2d_slice_with_step(self): source = numpy.random.randint(10, size=(10, 20)) new_data = numpy.random.randint(10, size=(2, 5)) slc = (slice(5, 10, 3), slice(5, 15, 2)) - arr = self.dac.fromarray(source) + arr = self.context.fromarray(source) source[slc] = new_data arr[slc] = new_data assert_array_equal(arr.toarray(), source) @@ -285,7 +285,7 @@ def test_3d_slice_ellipsis_with_step(self): source = numpy.random.randint(10, size=(5, 4, 5)) new_data = numpy.random.randint(10, size=(5, 2, 5)) slc = (Ellipsis, slice(None, None, 2), Ellipsis) - arr = self.dac.fromarray(source) + arr = self.context.fromarray(source) source[slc] = new_data arr[slc] = new_data assert_array_equal(arr.toarray(), source) diff --git a/distarray/dist/tests/test_maps.py b/distarray/dist/tests/test_maps.py index a0d86a3e..fd963899 100644 --- a/distarray/dist/tests/test_maps.py +++ b/distarray/dist/tests/test_maps.py @@ -7,7 +7,6 @@ from __future__ import division import unittest -import numpy from random import randrange from distarray.externals.six.moves import range @@ -146,7 +145,7 @@ def test_from_full_slice_1d(self): self.assertSequenceEqual(d1.maps[0].bounds, d0.maps[0].bounds) def test_from_full_slice_with_step_1d_0(self): - d0 = maps.Distribution.from_shape(context=self.ctx, shape=(15,)) + d0 = maps.Distribution.from_shape(context=self.context, shape=(15,)) s = (slice(None, None, 2),) d1 = d0.slice(s) @@ -157,7 +156,7 @@ def test_from_full_slice_with_step_1d_0(self): self.assertEqual(d1.maps[0].bounds[0][0], d0.maps[0].bounds[0][0]) def test_from_full_slice_with_step_1d_1(self): - d0 = maps.Distribution.from_shape(context=self.ctx, shape=(30,)) + d0 = maps.Distribution.from_shape(context=self.context, shape=(30,)) step = 4 s = (slice(4, None, step),)