From 49f5a863af284f0cbefa4d75ae0015fa5b3217a4 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 9 May 2014 11:10:16 -0500 Subject: [PATCH 1/4] Faster test_save_3d test. --- distarray/dist/tests/test_distributed_io.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/distarray/dist/tests/test_distributed_io.py b/distarray/dist/tests/test_distributed_io.py index b794acb4..eaf155d1 100644 --- a/distarray/dist/tests/test_distributed_io.py +++ b/distarray/dist/tests/test_distributed_io.py @@ -209,12 +209,7 @@ def test_save_3d(self): dist = {0: 'b', 1: 'c', 2: 'n'} distribution = Distribution.from_shape(self.dac, shape, dist=dist) - da = self.dac.empty(distribution) - - for i in range(shape[0]): - for j in range(shape[1]): - for k in range(shape[2]): - da[i, j, k] = source[i, j, k] + da = self.dac.fromarray(source, distribution) self.dac.save_hdf5(self.output_path, da, mode='w') with self.h5py.File(self.output_path, 'r') as fp: From 40dd179a449e62b5e6212726c8b1170de99d0719 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 9 May 2014 12:31:32 -0500 Subject: [PATCH 2/4] speed up test___init___ by using smaller data ((100, 100) -> (5, 5)) --- distarray/dist/tests/test_distarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index c184af92..1cdef6a5 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -329,7 +329,7 @@ def tearDown(self): self.context.close() def test___init__(self): - shape = (100, 100) + shape = (5, 5) distribution = Distribution.from_shape(self.context, shape, ('b', 'c')) da = DistArray(distribution, dtype=int) da.fill(42) From 01c799e2806da6f8bbf7d6797f428fde2a2dbc55 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 9 May 2014 12:45:00 -0500 Subject: [PATCH 3/4] Speed up test_from_global_dim_data_uu with smaller data 30 element array instead of 120. --- distarray/dist/tests/test_distarray.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index 1cdef6a5..85958418 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -209,11 +209,12 @@ def test_from_global_dim_data_bc(self): distarr[i, j] = i*cols + j las = distarr.get_localarrays() local_shapes = [la.local_shape for la in las] - self.assertSequenceEqual(local_shapes, [(3,5), (3,4), (2,5), (2,4)]) + self.assertSequenceEqual(local_shapes, + [(3, 5), (3, 4), (2, 5), (2, 4)]) def test_from_global_dim_data_uu(self): - rows = 6 - cols = 20 + rows = 3 + cols = 10 row_ixs = numpy.random.permutation(range(rows)) col_ixs = numpy.random.permutation(range(cols)) row_indices = [row_ixs[:rows//2], row_ixs[rows//2:]] From 67444ca3065fe1a84804f28efa911037d33a6372 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 12 May 2014 16:21:35 -0500 Subject: [PATCH 4/4] Test speed ups in test_distarray by reducing roundtrips. speed things up by replacing for loops over arrays with calls to .toarray(). Which still tests that the array can be indexed, but it pulls it to the client and does it all on the client instead of remotely. This change affects: * test_from_global_dim_data_irregular_block * test_from_global_dim_data_bu * test_from_global_dim_data_uu * test_from_global_dim_data_bc * test_irregular_block_assignment --- distarray/dist/tests/test_distarray.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index 85958418..4e4bc150 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -137,8 +137,7 @@ def test_from_global_dim_data_irregular_block(self): ) distribution = Distribution(self.context, glb_dim_data) distarr = DistArray(distribution, dtype=int) - for i in range(global_size): - distarr[i] = i + distarr.toarray() def test_from_global_dim_data_1d(self): total_size = 40 @@ -180,9 +179,7 @@ def test_from_global_dim_data_bu(self): ) distribution = Distribution(self.context, glb_dim_data) distarr = DistArray(distribution, dtype=int) - for i in range(rows): - for j in range(cols): - distarr[i, j] = i*cols + j + distarr.toarray() def test_from_global_dim_data_bc(self): """ Test creation of a block-cyclic array. """ @@ -204,17 +201,15 @@ def test_from_global_dim_data_bc(self): },) distribution = Distribution(self.context, global_dim_data) distarr = DistArray(distribution, dtype=int) - for i in range(rows): - for j in range(cols): - distarr[i, j] = i*cols + j + distarr.toarray() las = distarr.get_localarrays() local_shapes = [la.local_shape for la in las] self.assertSequenceEqual(local_shapes, [(3, 5), (3, 4), (2, 5), (2, 4)]) def test_from_global_dim_data_uu(self): - rows = 3 - cols = 10 + rows = 6 + cols = 20 row_ixs = numpy.random.permutation(range(rows)) col_ixs = numpy.random.permutation(range(cols)) row_indices = [row_ixs[:rows//2], row_ixs[rows//2:]] @@ -227,9 +222,7 @@ def test_from_global_dim_data_uu(self): ) distribution = Distribution(self.context, glb_dim_data) distarr = DistArray(distribution, dtype=int) - for i in range(rows): - for j in range(cols): - distarr[i, j] = i*cols + j + distarr.toarray() def test_global_dim_data_local_dim_data_equivalence(self): rows, cols = 5, 9 @@ -301,7 +294,6 @@ def test_global_dim_data_local_dim_data_equivalence(self): self.assertSequenceEqual(actual, expected) def test_irregular_block_assignment(self): - global_shape = (5, 9) global_dim_data = ( { 'dist_type': 'b', @@ -314,9 +306,7 @@ def test_irregular_block_assignment(self): ) distribution = Distribution(self.context, global_dim_data) distarr = DistArray(distribution, dtype=int) - for i in range(global_shape[0]): - for j in range(global_shape[1]): - distarr[i, j] = i + j + distarr.toarray() class TestDistArrayCreation(unittest.TestCase):