From b35561f4a30c27bda63fe7121a0a0133ed68ccec Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 12 Jun 2014 17:18:24 -0500 Subject: [PATCH 1/4] Fix var. See gh-issue-435. --- distarray/local/localarray.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/distarray/local/localarray.py b/distarray/local/localarray.py index b8dc3ea2..6e15066d 100644 --- a/distarray/local/localarray.py +++ b/distarray/local/localarray.py @@ -978,28 +978,28 @@ def mean_reducer(reduce_comm, larr, out, axes, dtype): def var_reducer(reduce_comm, larr, out, axes, dtype): """ Core reduction function for var.""" + temp = larr.copy() + temp.ndarray = temp.ndarray.astype(float) + + # We hold the intermediate means in `mean`. mean = empty_like(out, dtype=float) if out is not None else None mean = mean_reducer(reduce_comm, larr, mean, axes, dtype=float) - temp = empty_like(larr, dtype=float) + # Have to broadcast mean.ndarray to all ranks in this reduce_comm. + mean = reduce_comm.bcast(mean, root=0) - # Make mean.ndarray's shape broadcastable. - if mean is not None: - mean_shape = tuple(1 if axis in axes else s - for (axis, s) in enumerate(larr.ndarray.shape)) - mean.ndarray.shape = mean_shape - # Copy mean.ndarray into temp.ndarray - temp.ndarray[...] = mean.ndarray + mean_shape = [] + for (ax, s) in enumerate(larr.ndarray.shape): + mean_shape.append(1 if ax in axes else s) + mean_shape = tuple(mean_shape) - # have to broadcast mean.ndarray to all ranks in this reduce_comm. - reduce_comm.Bcast(temp.ndarray, root=0) + mean.ndarray.shape = mean_shape # Do the variance calculation. - temp.ndarray[...] = (larr.ndarray - temp.ndarray) ** 2 + temp.ndarray[...] = (larr.ndarray - mean.ndarray) ** 2 # Get the mean reduction of temp's data. mean_reducer(reduce_comm, temp, out, axes, dtype) - return out From ae3dd407b25dd775fe39d804694eef23eb7d2b33 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 12 Jun 2014 17:23:56 -0500 Subject: [PATCH 2/4] Add regression test for gh-issue-435 --- distarray/dist/tests/test_distarray.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index 2ffd7c9a..9c0dcd0c 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -595,6 +595,11 @@ def test_sum_4D_cyclic(self): assert_allclose(darr_sum.tondarray(), arr_sum) assert_allclose(darr.sum().tondarray(), arr.sum()) + def test_gh_435_regression_with_var(self): + dist = Distribution.from_shape(self.context, shape=(14,), dist=('b')) + darr = self.context.ones(dist) + darr.var() + class TestFromLocalArrays(ContextTestCase): From 231b6be84a472c8850f90bd8574177a8e4b12e9c Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 13 Jun 2014 11:38:39 -0500 Subject: [PATCH 3/4] Fix white space, pep8, and address @kwmsmith's comments. --- distarray/local/localarray.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/distarray/local/localarray.py b/distarray/local/localarray.py index 6e15066d..10eb9875 100644 --- a/distarray/local/localarray.py +++ b/distarray/local/localarray.py @@ -950,23 +950,27 @@ def _basic_reducer(reduce_comm, op, func, args, kwargs, out): reduce_comm.Reduce(local_reduce, out_ndarray, op=op, root=0) return out + def min_reducer(reduce_comm, larr, out, axes, dtype): """ Core reduction function for min.""" return _basic_reducer(reduce_comm, MPI.MIN, - larr.ndarray.min, - (), {'axis':axes}, out) + larr.ndarray.min, + (), {'axis': axes}, out) + def max_reducer(reduce_comm, larr, out, axes, dtype): """ Core reduction function for max.""" return _basic_reducer(reduce_comm, MPI.MAX, - larr.ndarray.max, - (), {'axis':axes}, out) + larr.ndarray.max, + (), {'axis': axes}, out) + def sum_reducer(reduce_comm, larr, out, axes, dtype): """ Core reduction function for sum.""" return _basic_reducer(reduce_comm, MPI.SUM, - larr.ndarray.sum, - (), {'axis':axes, 'dtype':dtype}, out) + larr.ndarray.sum, + (), {'axis': axes, 'dtype': dtype}, out) + def mean_reducer(reduce_comm, larr, out, axes, dtype): """ Core reduction function for mean.""" @@ -978,8 +982,7 @@ def mean_reducer(reduce_comm, larr, out, axes, dtype): def var_reducer(reduce_comm, larr, out, axes, dtype): """ Core reduction function for var.""" - temp = larr.copy() - temp.ndarray = temp.ndarray.astype(float) + temp = empty_like(larr, dtype=float) # We hold the intermediate means in `mean`. mean = empty_like(out, dtype=float) if out is not None else None From 88d7fac13dfc3986a56f5cae6e88e176dd67be94 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 13 Jun 2014 17:19:42 -0500 Subject: [PATCH 4/4] Add targets to regression test. --- distarray/dist/tests/test_distarray.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index 9c0dcd0c..8c666a1c 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -596,7 +596,8 @@ def test_sum_4D_cyclic(self): assert_allclose(darr.sum().tondarray(), arr.sum()) def test_gh_435_regression_with_var(self): - dist = Distribution.from_shape(self.context, shape=(14,), dist=('b')) + dist = Distribution.from_shape(self.context, shape=(14,), dist=('b'), + targets=range(4)) darr = self.context.ones(dist) darr.var()