From 0c3dc1c20190bdd75611a38e67228520f145f061 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Fri, 27 Jun 2014 18:22:01 -0500 Subject: [PATCH 1/8] Fix empty, ones, zeros for 0d. --- distarray/dist/context.py | 5 ++- distarray/dist/tests/test_distarray.py | 50 +++++++++++++++++++------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/distarray/dist/context.py b/distarray/dist/context.py index d808fb28..89f848aa 100644 --- a/distarray/dist/context.py +++ b/distarray/dist/context.py @@ -217,7 +217,10 @@ def _create_local(self, local_call, distribution, dtype): def create_local(local_call, ddpr, dtype, comm): from distarray.local.maps import Distribution local_call = local_call - dim_data = ddpr[comm.Get_rank()] + if len(ddpr) > 0: + dim_data = ddpr[comm.Get_rank()] + else: + dim_data = () distribution = Distribution(comm=comm, dim_data=dim_data) rval = local_call(distribution=distribution, dtype=dtype) return proxyize(rval) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index 2ed1c7de..0db91e4f 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -251,14 +251,6 @@ def test_all_ellipsis(self): assert_array_equal(arr[..., ..., ..., ...].toarray(), expected[..., ..., ..., ...]) - @unittest.skip("Waiting on 0d-array support.") - def test_0d_ellipsis(self): - shape = () - expected = numpy.random.randint(10, size=shape) - arr = self.context.fromarray(expected) - assert_array_equal(arr[...].toarray(), - expected[...]) - def test_resulting_slice(self): dist = Distribution.from_shape(self.context, (10, 20)) da = self.context.ones(dist) @@ -266,6 +258,13 @@ def test_resulting_slice(self): dc = db * 2 assert_array_equal(dc.toarray(), numpy.ones(dc.shape) * 2) + @unittest.skip('') + def test_0d_ellipsis(self): + shape = () + expected = numpy.random.randint(10, size=shape) + arr = self.context.fromarray(expected) + assert_array_equal(arr[...].toarray(), expected[...]) + class TestSetItemSlicing(ContextTestCase): @@ -606,15 +605,36 @@ def test_zeros(self): zero_ndarray = numpy.zeros(shape) assert_array_equal(zero_distarray.tondarray(), zero_ndarray) + def test_zeros_0d(self): + shape = () + distribution = Distribution.from_shape(self.context, shape) + zero_distarray = self.context.zeros(distribution) + zero_ndarray = numpy.zeros(shape) + assert_array_equal(zero_distarray.tondarray(), zero_ndarray) + def test_ones(self): shape = (16, 16) distribution = Distribution.from_shape(self.context, shape) - one_distarray = self.context.ones(distribution) - one_ndarray = numpy.ones(shape) - assert_array_equal(one_distarray.tondarray(), one_ndarray) + ones_distarray = self.context.ones(distribution) + ones_ndarray = numpy.ones(shape) + assert_array_equal(ones_distarray.tondarray(), ones_ndarray) + + def test_ones_0d(self): + shape = () + distribution = Distribution.from_shape(self.context, shape) + ones_distarray = self.context.ones(distribution) + ones_ndarray = numpy.ones(shape) + assert_array_equal(ones_distarray.tondarray(), ones_ndarray) def test_empty(self): - distribution = Distribution.from_shape(self.context, (16, 16)) + shape = (16, 16) + distribution = Distribution.from_shape(self.context, shape) + empty_distarray = self.context.empty(distribution) + self.assertEqual(empty_distarray.shape, distribution.shape) + + def test_empty_0d(self): + shape = () + distribution = Distribution.from_shape(self.context, shape) empty_distarray = self.context.empty(distribution) self.assertEqual(empty_distarray.shape, distribution.shape) @@ -624,6 +644,12 @@ def test_fromndarray(self): for (i, j), val in numpy.ndenumerate(ndarr): self.assertEqual(distarr[i, j], ndarr[i, j]) + @unittest.skip('') + def test_fromndarray_0d(self): + ndarr = numpy.array(42) + distarr = self.context.fromarray(ndarr) + assert_array_equal(ndarr, distarr.toarray()) + def test_grid_rank(self): # regression test for issue #235 d = Distribution.from_shape(self.context, (4, 4, 4), From 02f4d3258fc2b0e1249d4959a7bcc1a7a7db775c Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Fri, 27 Jun 2014 18:28:25 -0500 Subject: [PATCH 2/8] Fix fromndarray for 0d arrays. --- distarray/dist/distarray.py | 7 +++++-- distarray/dist/tests/test_distarray.py | 1 - 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/distarray/dist/distarray.py b/distarray/dist/distarray.py index 830a8c84..24849610 100644 --- a/distarray/dist/distarray.py +++ b/distarray/dist/distarray.py @@ -21,7 +21,7 @@ import numpy as np -import distarray +import distarray.local from distarray.metadata_utils import sanitize_indices from distarray.dist.maps import Distribution from distarray.utils import _raise_nie @@ -215,7 +215,10 @@ def _set_view(self, index, value): def set_view(arr, index, value, ddpr, comm): from distarray.local.localarray import LocalArray from distarray.local.maps import Distribution - dim_data = ddpr[comm.Get_rank()] + if len(ddpr) > 0: + dim_data = ddpr[comm.Get_rank()] + else: + dim_data = [] dist = Distribution(comm=comm, dim_data=dim_data) if isinstance(value, LocalArray): arr.global_index[index] = value.ndarray diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index 0db91e4f..e3c7df81 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -644,7 +644,6 @@ def test_fromndarray(self): for (i, j), val in numpy.ndenumerate(ndarr): self.assertEqual(distarr[i, j], ndarr[i, j]) - @unittest.skip('') def test_fromndarray_0d(self): ndarr = numpy.array(42) distarr = self.context.fromarray(ndarr) From 3ed2528b601c1d1686317fa94279c7b3e76c8c94 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Fri, 27 Jun 2014 18:33:00 -0500 Subject: [PATCH 3/8] Test 0d array getitem with empty tuple. --- distarray/dist/tests/test_distarray.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index e3c7df81..4ad84419 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -258,7 +258,13 @@ def test_resulting_slice(self): dc = db * 2 assert_array_equal(dc.toarray(), numpy.ones(dc.shape) * 2) - @unittest.skip('') + def test_0d_empty_tuple(self): + shape = () + expected = numpy.random.randint(10, size=shape) + arr = self.context.fromarray(expected) + assert_array_equal(arr[()], expected[()]) + + @unittest.skip('Ellipsis not yet supported for 0d.') def test_0d_ellipsis(self): shape = () expected = numpy.random.randint(10, size=shape) From bf16a855f62a1cbf7bc392e22460a142ab1b5de8 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 30 Jun 2014 18:09:06 -0500 Subject: [PATCH 4/8] Unskip 0d ellipsis array test. Fails. --- distarray/dist/tests/test_distarray.py | 1 - 1 file changed, 1 deletion(-) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index 4ad84419..a8ab6b47 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -264,7 +264,6 @@ def test_0d_empty_tuple(self): arr = self.context.fromarray(expected) assert_array_equal(arr[()], expected[()]) - @unittest.skip('Ellipsis not yet supported for 0d.') def test_0d_ellipsis(self): shape = () expected = numpy.random.randint(10, size=shape) From a83fc28c845ad24299fe23fedf3eae063f27dc7f Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 30 Jun 2014 18:14:40 -0500 Subject: [PATCH 5/8] Fix 0d array test. Also, raise an exception when, locally, distribution.shape != buf.shape. --- distarray/local/localarray.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/distarray/local/localarray.py b/distarray/local/localarray.py index f7432e6a..83c7869a 100644 --- a/distarray/local/localarray.py +++ b/distarray/local/localarray.py @@ -111,8 +111,10 @@ def __init__(self, distribution, dtype=None, buf=None): if buf is None: self.ndarray = np.empty(self.local_shape, dtype=dtype) else: - mv = memoryview(buf) - self.ndarray = np.asarray(mv, dtype=dtype) + self.ndarray = np.asarray(buf, dtype=dtype) + if distribution.local_shape != self.ndarray.shape: + msg = "distribution shape must equal buf shape." + raise RuntimeError(msg) # We pass a view of self.ndarray because we want the # GlobalIndex object to be able to change the LocalArray From cb33fd354bd4e0209b6956ab7778a240d3d438f9 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 30 Jun 2014 18:16:19 -0500 Subject: [PATCH 6/8] Fix another test that my "raise" uncovered. --- ...paralleltest_distributed_array_protocol.py | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/distarray/local/tests/paralleltest_distributed_array_protocol.py b/distarray/local/tests/paralleltest_distributed_array_protocol.py index 4d8c46d8..05c38454 100644 --- a/distarray/local/tests/paralleltest_distributed_array_protocol.py +++ b/distarray/local/tests/paralleltest_distributed_array_protocol.py @@ -166,15 +166,32 @@ class TestDapLopsided(DapValidatorMixin, MpiTestCase): comm_size = 2 def setUp(self): + global_size = 50 if self.comm.Get_rank() == 0: - arr = np.arange(20) + local_size = 20 + arr = np.arange(local_size) + dim_data = ({ + 'dist_type': 'b', + 'size': global_size, + 'proc_grid_size': 2, + 'proc_grid_rank': 0, + 'start': 0, + 'stop': local_size, + },) elif self.comm.Get_rank() == 1: - arr = np.arange(30) - - d = Distribution.from_shape(comm=self.comm, shape=(50,), - dist={0: 'b', 1: 'n'}, grid_shape=(2,)) - - self.larr = LocalArray(d, dtype='float64', buf=arr) + local_size = 30 + arr = np.arange(local_size) + dim_data = ({ + 'dist_type': 'b', + 'size': global_size, + 'proc_grid_size': 2, + 'proc_grid_rank': 1, + 'start': 20, + 'stop': global_size, + },) + + d = Distribution(comm=self.comm, dim_data=dim_data) + self.larr = LocalArray(d, buf=arr) def test_values(self): if self.comm.Get_rank() == 0: From 3d9c4361d7e17239d8eef5ae0e53da8ebf8000d5 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Mon, 30 Jun 2014 18:28:48 -0500 Subject: [PATCH 7/8] Test setting 0d arrays. --- distarray/dist/tests/test_distarray.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index a8ab6b47..fdc4dcc4 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -397,6 +397,18 @@ def test_set_DistArray_slice(self): db = self.context.zeros(dist) da[...] = db + def test_set_0d_slice(self): + expected = numpy.array(33) + arr = self.context.fromarray(expected) + + val0 = 55 + arr[...] = val0 + assert_array_equal(arr[...].toarray(), numpy.array(val0)) + + val1 = 99 + arr[()] = val1 + assert_array_equal(arr[...].toarray(), numpy.array(val1)) + class TestDistArrayCreationFromGlobalDimData(ContextTestCase): From 38ae724b00d492f42341f2e76531d6fc76a40888 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 1 Jul 2014 11:51:21 -0500 Subject: [PATCH 8/8] Check for empty ddpr the same way everywhere. --- distarray/dist/context.py | 6 +++--- distarray/dist/distarray.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/distarray/dist/context.py b/distarray/dist/context.py index 81a4f4ad..20d67347 100644 --- a/distarray/dist/context.py +++ b/distarray/dist/context.py @@ -216,10 +216,10 @@ def _create_local(self, local_call, distribution, dtype): """Creates LocalArrays with the method named in `local_call`.""" def create_local(local_call, ddpr, dtype, comm): from distarray.local.maps import Distribution - if len(ddpr) > 0: - dim_data = ddpr[comm.Get_rank()] - else: + if len(ddpr) == 0: dim_data = () + else: + dim_data = ddpr[comm.Get_rank()] local_call = eval(local_call) distribution = Distribution(comm=comm, dim_data=dim_data) rval = local_call(distribution=distribution, dtype=dtype) diff --git a/distarray/dist/distarray.py b/distarray/dist/distarray.py index c0ce9f9c..a552e094 100644 --- a/distarray/dist/distarray.py +++ b/distarray/dist/distarray.py @@ -215,10 +215,10 @@ def _set_view(self, index, value): def set_view(arr, index, value, ddpr, comm): from distarray.local.localarray import LocalArray from distarray.local.maps import Distribution - if len(ddpr) > 0: - dim_data = ddpr[comm.Get_rank()] + if len(ddpr) == 0: + dim_data = () else: - dim_data = [] + dim_data = ddpr[comm.Get_rank()] dist = Distribution(comm=comm, dim_data=dim_data) if isinstance(value, LocalArray): arr.global_index[index] = value.ndarray