From 713c894f9d3680b9457fbcc03eaf4a390d321600 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 11:40:52 -0500 Subject: [PATCH 01/19] map_from_dim_datas -> map_from_dim_data_per_rank --- distarray/client_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distarray/client_map.py b/distarray/client_map.py index 2cee09c1..9946f3b6 100644 --- a/distarray/client_map.py +++ b/distarray/client_map.py @@ -73,7 +73,7 @@ def choose_map(dist_type): return cls_from_dist_type[dist_type] -def map_from_dim_datas(dim_datas): +def map_from_dim_data_per_rank(dim_datas): """ Generates a ClientMap instance from a santized sequence of dim_data dictionaries. @@ -411,7 +411,7 @@ def from_dim_data(cls, context, dim_datas): if len(dim_data_per_dim) != self.ndim: raise ValueError("Inconsistent dimensions.") - self.maps = [map_from_dim_datas(ddpd) for ddpd in dim_data_per_dim] + self.maps = [map_from_dim_data_per_rank(ddpd) for ddpd in dim_data_per_dim] return self From 02c4885f02724ed6ae91565d52c597eaceaa067c Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 11:41:56 -0500 Subject: [PATCH 02/19] Fix spelling errors in map_from_dim_data_per_rank --- distarray/client_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distarray/client_map.py b/distarray/client_map.py index 9946f3b6..6e37e55f 100644 --- a/distarray/client_map.py +++ b/distarray/client_map.py @@ -74,7 +74,7 @@ def choose_map(dist_type): def map_from_dim_data_per_rank(dim_datas): - """ Generates a ClientMap instance from a santized sequence of dim_data + """ Generates a ClientMap instance from a sanitized sequence of dim_data dictionaries. Parameters @@ -91,7 +91,7 @@ def map_from_dim_data_per_rank(dim_datas): An instance of a subclass of MapBase. """ - # check that all proccesses / ranks are accounted for. + # check that all processes / ranks are accounted for. proc_ranks = sorted(dd['proc_grid_rank'] for dd in dim_datas) if proc_ranks != list(range(len(dim_datas))): msg = "Ranks of processes (%r) not consistent." From 2686323c8c74fec597013546aae69e736de9d564 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 11:44:16 -0500 Subject: [PATCH 03/19] dim_datas -> dim_data_per_rank in map_from_ddpr --- distarray/client_map.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/distarray/client_map.py b/distarray/client_map.py index 6e37e55f..0b9a09cc 100644 --- a/distarray/client_map.py +++ b/distarray/client_map.py @@ -73,13 +73,13 @@ def choose_map(dist_type): return cls_from_dist_type[dist_type] -def map_from_dim_data_per_rank(dim_datas): +def map_from_dim_data_per_rank(dim_data_per_rank): """ Generates a ClientMap instance from a sanitized sequence of dim_data dictionaries. Parameters ---------- - dim_datas : sequence of dictionaries + dim_data_per_rank : sequence of dictionaries Each dictionary is a "dimension dictionary" from the distributed array protocol, one per process in this dimension of the process grid. The dimension dictionaries shall all have the same keys and values for @@ -92,16 +92,17 @@ def map_from_dim_data_per_rank(dim_datas): """ # check that all processes / ranks are accounted for. - proc_ranks = sorted(dd['proc_grid_rank'] for dd in dim_datas) - if proc_ranks != list(range(len(dim_datas))): + proc_ranks = sorted(dd['proc_grid_rank'] for dd in dim_data_per_rank) + if proc_ranks != list(range(len(dim_data_per_rank))): msg = "Ranks of processes (%r) not consistent." raise ValueError(msg % proc_ranks) - # Sort dim_datas according to proc_grid_rank. - dim_datas = sorted(dim_datas, key=lambda d: d['proc_grid_rank']) + # Sort dim_data_per_rank according to proc_grid_rank. + dim_data_per_rank = sorted(dim_data_per_rank, + key=lambda d: d['proc_grid_rank']) - dist_type = dim_datas[0]['dist_type'] + dist_type = dim_data_per_rank[0]['dist_type'] map_class = choose_map(dist_type) - return map_class.from_dim_data(dim_datas) + return map_class.from_dim_data(dim_data_per_rank) def map_from_global_dim_dict(global_dim_dict): From 37e13e4465d7be7bfe9e3321b92530d3e39a06be Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 11:48:25 -0500 Subject: [PATCH 04/19] from_dim_data -> from_dim_data_per_rank in Maps --- distarray/client_map.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/distarray/client_map.py b/distarray/client_map.py index 0b9a09cc..9c2bc3fd 100644 --- a/distarray/client_map.py +++ b/distarray/client_map.py @@ -102,7 +102,7 @@ def map_from_dim_data_per_rank(dim_data_per_rank): dist_type = dim_data_per_rank[0]['dist_type'] map_class = choose_map(dist_type) - return map_class.from_dim_data(dim_data_per_rank) + return map_class.from_dim_data_per_rank(dim_data_per_rank) def map_from_global_dim_dict(global_dim_dict): @@ -165,7 +165,7 @@ def from_global_dim_dict(cls, glb_dim_dict): return cls(size, grid_size=1) @classmethod - def from_dim_data(cls, dim_data_seq): + def from_dim_data_per_rank(cls, dim_data_seq): if len(dim_data_seq) != 1: msg = ("Number of dimension dictionaries " "non-unitary for non-distributed dimension.") @@ -220,7 +220,7 @@ def from_global_dim_dict(cls, glb_dim_dict): return self @classmethod - def from_dim_data(cls, dim_data_seq): + def from_dim_data_per_rank(cls, dim_data_seq): self = cls.__new__(cls) dd = dim_data_seq[0] if dd['dist_type'] != 'b': @@ -288,7 +288,7 @@ def from_global_dim_dict(cls, glb_dim_dict): return cls(size, grid_size, block_size) @classmethod - def from_dim_data(cls, dim_data_seq): + def from_dim_data_per_rank(cls, dim_data_seq): dd = dim_data_seq[0] if dd['dist_type'] != 'c': msg = "Wrong dist_type (%r) for cyclic map." @@ -336,7 +336,7 @@ def from_global_dim_dict(cls, glb_dim_dict): return cls(size, grid_size, indices=indices) @classmethod - def from_dim_data(cls, dim_data_seq): + def from_dim_data_per_rank(cls, dim_data_seq): dd = dim_data_seq[0] if dd['dist_type'] != 'u': msg = "Wrong dist_type (%r) for unstructured map." From bf25d73ef809dacfe206455af25c6c403443f352 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 11:49:28 -0500 Subject: [PATCH 05/19] from_dim_data -> from_dim_data_per_rank in Distribution --- distarray/client.py | 2 +- distarray/client_map.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distarray/client.py b/distarray/client.py index e8962175..613d0921 100644 --- a/distarray/client.py +++ b/distarray/client.py @@ -75,7 +75,7 @@ def _make_mdmap_from_local_dimdata(local_name, context): dim_data_name = context._generate_key() context._execute(_DIMDATAS.format(local_name=local_name, dim_data_name=dim_data_name)) dim_datas = context._pull(dim_data_name) - return Distribution.from_dim_data(context, dim_datas) + return Distribution.from_dim_data_per_rank(context, dim_datas) def _get_attribute(context, key, name): local_key = context._generate_key() diff --git a/distarray/client_map.py b/distarray/client_map.py index 9c2bc3fd..0e3f7464 100644 --- a/distarray/client_map.py +++ b/distarray/client_map.py @@ -388,7 +388,7 @@ class Distribution(object): """ @classmethod - def from_dim_data(cls, context, dim_datas): + def from_dim_data_per_rank(cls, context, dim_datas): """ Creates a Distribution from a sequence of `dim_data` dictionary tuples from each LocalArray. """ From b0bfbbe9ebf70eda02ca8c9382c10c755c9929e9 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 11:52:08 -0500 Subject: [PATCH 06/19] dim_data_seq -> dim_data_per_rank in Maps --- distarray/client_map.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/distarray/client_map.py b/distarray/client_map.py index 0e3f7464..889f42ca 100644 --- a/distarray/client_map.py +++ b/distarray/client_map.py @@ -165,12 +165,12 @@ def from_global_dim_dict(cls, glb_dim_dict): return cls(size, grid_size=1) @classmethod - def from_dim_data_per_rank(cls, dim_data_seq): - if len(dim_data_seq) != 1: + def from_dim_data_per_rank(cls, dim_data_per_rank): + if len(dim_data_per_rank) != 1: msg = ("Number of dimension dictionaries " "non-unitary for non-distributed dimension.") raise ValueError(msg) - dd = dim_data_seq[0] + dd = dim_data_per_rank[0] if dd['dist_type'] != 'n': msg = "Wrong dist_type (%r) for non-distributed map." raise ValueError(msg % dd['dist_type']) @@ -220,19 +220,19 @@ def from_global_dim_dict(cls, glb_dim_dict): return self @classmethod - def from_dim_data_per_rank(cls, dim_data_seq): + def from_dim_data_per_rank(cls, dim_data_per_rank): self = cls.__new__(cls) - dd = dim_data_seq[0] + dd = dim_data_per_rank[0] if dd['dist_type'] != 'b': msg = "Wrong dist_type (%r) for block map." raise ValueError(msg % dd['dist_type']) self.size = dd['size'] self.grid_size = dd['proc_grid_size'] - if self.grid_size != len(dim_data_seq): + if self.grid_size != len(dim_data_per_rank): msg = ("Number of dimension dictionaries (%r)" "inconsistent with proc_grid_size (%r).") - raise ValueError(msg % (len(dim_data_seq), self.grid_size)) - self.bounds = [(d['start'], d['stop']) for d in dim_data_seq] + raise ValueError(msg % (len(dim_data_per_rank), self.grid_size)) + self.bounds = [(d['start'], d['stop']) for d in dim_data_per_rank] self.boundary_padding, self.comm_padding = dd.get('padding', (0, 0)) return self @@ -288,17 +288,17 @@ def from_global_dim_dict(cls, glb_dim_dict): return cls(size, grid_size, block_size) @classmethod - def from_dim_data_per_rank(cls, dim_data_seq): - dd = dim_data_seq[0] + def from_dim_data_per_rank(cls, dim_data_per_rank): + dd = dim_data_per_rank[0] if dd['dist_type'] != 'c': msg = "Wrong dist_type (%r) for cyclic map." raise ValueError(msg % dd['dist_type']) size = dd['size'] grid_size = dd['proc_grid_size'] - if grid_size != len(dim_data_seq): + if grid_size != len(dim_data_per_rank): msg = ("Number of dimension dictionaries (%r)" "inconsistent with proc_grid_size (%r).") - raise ValueError(msg % (len(dim_data_seq), grid_size)) + raise ValueError(msg % (len(dim_data_per_rank), grid_size)) block_size = dd.get('block_size', 1) return cls(size, grid_size, block_size) @@ -336,18 +336,18 @@ def from_global_dim_dict(cls, glb_dim_dict): return cls(size, grid_size, indices=indices) @classmethod - def from_dim_data_per_rank(cls, dim_data_seq): - dd = dim_data_seq[0] + def from_dim_data_per_rank(cls, dim_data_per_rank): + dd = dim_data_per_rank[0] if dd['dist_type'] != 'u': msg = "Wrong dist_type (%r) for unstructured map." raise ValueError(msg % dd['dist_type']) size = dd['size'] grid_size = dd['proc_grid_size'] - if grid_size != len(dim_data_seq): + if grid_size != len(dim_data_per_rank): msg = ("Number of dimension dictionaries (%r)" "inconsistent with proc_grid_size (%r).") - raise ValueError(msg % (len(dim_data_seq), grid_size)) - indices = [dd['indices'] for dd in dim_data_seq] + raise ValueError(msg % (len(dim_data_per_rank), grid_size)) + indices = [dd['indices'] for dd in dim_data_per_rank] return cls(size, grid_size, indices=indices) def __init__(self, size, grid_size, indices=None): From 8d390965d2ca2527e22f470f23c8b8a5b088e6d8 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 11:53:27 -0500 Subject: [PATCH 07/19] dim_datas -> dim_data_per_rank in Distribution --- distarray/client_map.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/distarray/client_map.py b/distarray/client_map.py index 889f42ca..b66463e1 100644 --- a/distarray/client_map.py +++ b/distarray/client_map.py @@ -388,13 +388,13 @@ class Distribution(object): """ @classmethod - def from_dim_data_per_rank(cls, context, dim_datas): + def from_dim_data_per_rank(cls, context, dim_data_per_rank): """ Creates a Distribution from a sequence of `dim_data` dictionary tuples from each LocalArray. """ self = cls.__new__(cls) - dd0 = dim_datas[0] + dd0 = dim_data_per_rank[0] self.context = context self.shape = tuple(dd['size'] for dd in dd0) self.ndim = len(dd0) @@ -403,11 +403,12 @@ def from_dim_data_per_rank(cls, context, dim_datas): validate_grid_shape(self.grid_shape, self.dist, len(context.targets)) - coords = [tuple(d['proc_grid_rank'] for d in dd) for dd in dim_datas] + coords = [tuple(d['proc_grid_rank'] for d in dd) for dd in + dim_data_per_rank] self.rank_from_coords = {c: r for (r, c) in enumerate(coords)} dim_data_per_dim = [_compactify_dicts(dict_tuple) - for dict_tuple in zip(*dim_datas)] + for dict_tuple in zip(*dim_data_per_rank)] if len(dim_data_per_dim) != self.ndim: raise ValueError("Inconsistent dimensions.") From 595a1cf5787dc49ec4105f4d0b331bba66ebd6a0 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 11:55:12 -0500 Subject: [PATCH 08/19] get_local_dim_datas -> get_dim_data_per_rank in client_map.py. --- distarray/client_map.py | 2 +- distarray/context.py | 2 +- distarray/tests/test_client.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distarray/client_map.py b/distarray/client_map.py index b66463e1..5d951a7a 100644 --- a/distarray/client_map.py +++ b/distarray/client_map.py @@ -483,7 +483,7 @@ def owning_targets(self, idxs): """ return [self.context.targets[r] for r in self.owning_ranks(idxs)] - def get_local_dim_datas(self): + def get_dim_data_per_rank(self): dds = [enumerate(m.get_dimdicts()) for m in self.maps] cart_dds = product(*dds) coord_and_dd = [zip(*cdd) for cdd in cart_dds] diff --git a/distarray/context.py b/distarray/context.py index ae887a0d..daca4a22 100644 --- a/distarray/context.py +++ b/distarray/context.py @@ -321,7 +321,7 @@ def from_global_dim_data(self, global_dim_data, dtype=float): """ # global_dim_data is a sequence of dictionaries, one per dimension. mdmap = Distribution(self, global_dim_data) - dim_data_per_rank = mdmap.get_local_dim_datas() + dim_data_per_rank = mdmap.get_dim_data_per_rank() if len(self.targets) != len(dim_data_per_rank): errmsg = "`dim_data_per_rank` must contain a dim_data for every rank." diff --git a/distarray/tests/test_client.py b/distarray/tests/test_client.py index 0467024f..b7c43677 100644 --- a/distarray/tests/test_client.py +++ b/distarray/tests/test_client.py @@ -216,7 +216,7 @@ def test_global_dim_data_local_dim_data_equivalence(self): }, ) mdmap = Distribution(self.context, glb_dim_data) - actual = mdmap.get_local_dim_datas() + actual = mdmap.get_dim_data_per_rank() expected = [ ({'block_size': 2, From 9df1b54a4782f19233d6645f04fad510c11ced03 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 12:01:37 -0500 Subject: [PATCH 09/19] dim_datas -> dim_data_per_rank in client.py --- distarray/client.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/distarray/client.py b/distarray/client.py index 613d0921..9d87b541 100644 --- a/distarray/client.py +++ b/distarray/client.py @@ -67,15 +67,16 @@ def is_LocalArray(typestring): return result -_DIMDATAS = """ +_DIM_DATA_PER_RANK = """ {dim_data_name} = {local_name}.dim_data """ def _make_mdmap_from_local_dimdata(local_name, context): dim_data_name = context._generate_key() - context._execute(_DIMDATAS.format(local_name=local_name, dim_data_name=dim_data_name)) - dim_datas = context._pull(dim_data_name) - return Distribution.from_dim_data_per_rank(context, dim_datas) + context._execute(_DIM_DATA_PER_RANK.format(local_name=local_name, + dim_data_name=dim_data_name)) + dim_data_per_rank = context._pull(dim_data_name) + return Distribution.from_dim_data_per_rank(context, dim_data_per_rank) def _get_attribute(context, key, name): local_key = context._generate_key() From 8e836096f57e533418677e6efb53e3af55265a94 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 12:06:39 -0500 Subject: [PATCH 10/19] dim_datas -> dim_data_per_rank in tests --- distarray/local/tests/paralleltest_io.py | 31 ++++++++++++---------- distarray/local/tests/paralleltest_maps.py | 11 ++++---- distarray/tests/test_distributed_io.py | 12 ++++----- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/distarray/local/tests/paralleltest_io.py b/distarray/local/tests/paralleltest_io.py index 3e832c6e..f44f9784 100644 --- a/distarray/local/tests/paralleltest_io.py +++ b/distarray/local/tests/paralleltest_io.py @@ -161,23 +161,26 @@ def tearDown(self): os.remove(self.output_path) def test_load_bn(self): - dim_datas = bn_test_data - la = load_npy(self.output_path, dim_datas[self.rank], comm=self.comm) + dim_data_per_rank = bn_test_data + la = load_npy(self.output_path, dim_data_per_rank[self.rank], + comm=self.comm) assert_equal(la, self.expected[numpy.newaxis, self.rank]) def test_load_nc(self): - dim_datas = nc_test_data + dim_data_per_rank = nc_test_data expected_slices = [(slice(None), slice(0, None, 2)), (slice(None), slice(1, None, 2))] - la = load_npy(self.output_path, dim_datas[self.rank], comm=self.comm) + la = load_npy(self.output_path, dim_data_per_rank[self.rank], + comm=self.comm) assert_equal(la, self.expected[expected_slices[self.rank]]) def test_load_nu(self): - dim_datas = nu_test_data - expected_indices = [dd[1]['indices'] for dd in dim_datas] + dim_data_per_rank = nu_test_data + expected_indices = [dd[1]['indices'] for dd in dim_data_per_rank] - la = load_npy(self.output_path, dim_datas[self.rank], comm=self.comm) + la = load_npy(self.output_path, dim_data_per_rank[self.rank], + comm=self.comm) assert_equal(la, self.expected[:, expected_indices[self.rank]]) @@ -253,18 +256,18 @@ def tearDown(self): os.remove(self.output_path) def test_load_bn(self): - dim_datas = bn_test_data - la = load_hdf5(self.output_path, dim_datas[self.rank], + dim_data_per_rank = bn_test_data + la = load_hdf5(self.output_path, dim_data_per_rank[self.rank], key=self.key, comm=self.comm) with self.h5py.File(self.output_path, 'r', driver='mpio', comm=self.comm) as fp: assert_equal(la, self.expected[numpy.newaxis, self.rank]) def test_load_nc(self): - dim_datas = nc_test_data + dim_data_per_rank = nc_test_data expected_slices = [(slice(None), slice(0, None, 2)), (slice(None), slice(1, None, 2))] - la = load_hdf5(self.output_path, dim_datas[self.rank], + la = load_hdf5(self.output_path, dim_data_per_rank[self.rank], key=self.key, comm=self.comm) with self.h5py.File(self.output_path, 'r', driver='mpio', comm=self.comm) as fp: @@ -272,9 +275,9 @@ def test_load_nc(self): assert_equal(la, self.expected[expected_slice]) def test_load_nu(self): - dim_datas = nu_test_data - expected_indices = [dd[1]['indices'] for dd in dim_datas] - la = load_hdf5(self.output_path, dim_datas[self.rank], + dim_data_per_rank = nu_test_data + expected_indices = [dd[1]['indices'] for dd in dim_data_per_rank] + la = load_hdf5(self.output_path, dim_data_per_rank[self.rank], key=self.key, comm=self.comm) with self.h5py.File(self.output_path, 'r', driver='mpio', comm=self.comm) as fp: diff --git a/distarray/local/tests/paralleltest_maps.py b/distarray/local/tests/paralleltest_maps.py index 532ff971..4c33774b 100644 --- a/distarray/local/tests/paralleltest_maps.py +++ b/distarray/local/tests/paralleltest_maps.py @@ -162,9 +162,10 @@ def test_block(self): "size": 16, } dd1 = (dim10, dim11) - dim_datas = (dd0, dd1) + dim_data_per_rank = (dd0, dd1) - d0 = Distribution(dim_datas[self.comm.Get_rank()], comm=self.comm) + d0 = Distribution(dim_data_per_rank[self.comm.Get_rank()], + comm=self.comm) d1 = Distribution.from_shape((16, 16), dist={0: 'b'}, grid_shape=(2, 1), comm=self.comm) self.assert_alike(d0, d1) @@ -197,10 +198,10 @@ def test_cyclic(self): } dd1 = (dim10, dim11) - dim_datas = (dd0, dd1) + dim_data_per_rank = (dd0, dd1) - - larr = Distribution(dim_datas[self.comm.Get_rank()], comm=self.comm) + larr = Distribution(dim_data_per_rank[self.comm.Get_rank()], + comm=self.comm) expected = Distribution.from_shape((16, 16), dist={1: 'c'}, grid_shape=(1, 2), comm=self.comm) diff --git a/distarray/tests/test_distributed_io.py b/distarray/tests/test_distributed_io.py index cd5528ac..bfa967c7 100644 --- a/distarray/tests/test_distributed_io.py +++ b/distarray/tests/test_distributed_io.py @@ -150,22 +150,22 @@ def tearDown(self): self.dac.close() def test_load_bn(self): - dim_datas = bn_test_data - da = self.dac.load_npy(self.output_path, dim_datas) + dim_data_per_rank = bn_test_data + da = self.dac.load_npy(self.output_path, dim_data_per_rank) for i in range(da.shape[0]): for j in range(da.shape[1]): self.assertEqual(da[i, j], self.expected[i, j]) def test_load_nc(self): - dim_datas = nc_test_data - da = self.dac.load_npy(self.output_path, dim_datas) + dim_data_per_rank = nc_test_data + da = self.dac.load_npy(self.output_path, dim_data_per_rank) for i in range(da.shape[0]): for j in range(da.shape[1]): self.assertEqual(da[i, j], self.expected[i, j]) def test_load_nu(self): - dim_datas = nu_test_data - da = self.dac.load_npy(self.output_path, dim_datas) + dim_data_per_rank = nu_test_data + da = self.dac.load_npy(self.output_path, dim_data_per_rank) for i in range(da.shape[0]): for j in range(da.shape[1]): self.assertEqual(da[i, j], self.expected[i, j]) From f7f086dcf93873ef30a5452fcf80fba1c661d1d7 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 12:16:18 -0500 Subject: [PATCH 11/19] ddpp -> ddpr in a test --- distarray/local/tests/paralleltest_localarray.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/distarray/local/tests/paralleltest_localarray.py b/distarray/local/tests/paralleltest_localarray.py index 50d35355..d160c699 100644 --- a/distarray/local/tests/paralleltest_localarray.py +++ b/distarray/local/tests/paralleltest_localarray.py @@ -341,7 +341,7 @@ def test_pack_unpack_index(self): class TestLocalArrayMethods(MpiTestCase): - ddpp = [ + ddpr = [ ({'dist_type': 'c', 'block_size': 1, 'size': 4, @@ -413,7 +413,7 @@ def test_copy_bn(self): assert_localarrays_equal(a, b, check_dtype=True) def test_copy_cbc(self): - distribution = Distribution(self.ddpp[self.comm.Get_rank()], + distribution = Distribution(self.ddpr[self.comm.Get_rank()], comm=self.comm) a = LocalArray(distribution, dtype=np.int_) a.fill(12) @@ -432,7 +432,7 @@ def test_astype_bn(self): def test_astype_cbc(self): new_dtype = np.int8 - d = Distribution(self.ddpp[self.comm.Get_rank()], comm=self.comm) + d = Distribution(self.ddpr[self.comm.Get_rank()], comm=self.comm) a = LocalArray(d, dtype=np.int32) a.fill(12) b = a.astype(new_dtype) From 882a9bb9ae15d02824f7d3beb28b15b392883e7b Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 12:17:48 -0500 Subject: [PATCH 12/19] dim_data_name -> ddpr_name in client.py --- distarray/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/distarray/client.py b/distarray/client.py index 9d87b541..851654fc 100644 --- a/distarray/client.py +++ b/distarray/client.py @@ -68,14 +68,14 @@ def is_LocalArray(typestring): return result _DIM_DATA_PER_RANK = """ -{dim_data_name} = {local_name}.dim_data +{ddpr_name} = {local_name}.dim_data """ def _make_mdmap_from_local_dimdata(local_name, context): - dim_data_name = context._generate_key() + ddpr_name = context._generate_key() context._execute(_DIM_DATA_PER_RANK.format(local_name=local_name, - dim_data_name=dim_data_name)) - dim_data_per_rank = context._pull(dim_data_name) + ddpr_name=ddpr_name)) + dim_data_per_rank = context._pull(ddpr_name) return Distribution.from_dim_data_per_rank(context, dim_data_per_rank) def _get_attribute(context, key, name): From 81b36b84158a190420d1f54c9e50d6023d04a813 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 12:37:41 -0500 Subject: [PATCH 13/19] Fix a spelling mistake in a docstring. --- distarray/client_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distarray/client_map.py b/distarray/client_map.py index 2cee09c1..a7ba63a3 100644 --- a/distarray/client_map.py +++ b/distarray/client_map.py @@ -61,7 +61,7 @@ def _compactify_dicts(dicts): # --------------------------------------------------------------------------- def choose_map(dist_type): - """Choose a map classe given one of the distribution types.""" + """Choose a map class given one of the distribution types.""" cls_from_dist_type = { 'b': BlockMap, 'c': BlockCyclicMap, From 1dc40e6f6ba49dbee03dbcc5000a24a47e50ca88 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 12:42:11 -0500 Subject: [PATCH 14/19] mdmap -> distribution in DistArray.__init__ param --- distarray/client.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/distarray/client.py b/distarray/client.py index e8962175..066003ce 100644 --- a/distarray/client.py +++ b/distarray/client.py @@ -88,23 +88,22 @@ class DistArray(object): __array_priority__ = 20.0 - def __init__(self, mdmap, dtype): - """ Creates a new empty distarray according to the multi-dimensional - map given. - """ + def __init__(self, distribution, dtype): + """Creates an empty DistArray according to the `distribution` given.""" # FIXME: code duplication with context.py. - ctx = mdmap.context + ctx = distribution.context # FIXME: this is bad... comm_name = ctx._comm_key # FIXME: and this is bad... da_key = ctx._generate_key() - names = ctx._key_and_push(mdmap.shape, mdmap.dist, mdmap.grid_shape, dtype) + names = ctx._key_and_push(distribution.shape, distribution.dist, + distribution.grid_shape, dtype) shape_name, dist_name, grid_shape_name, dtype_name = names cmd = ('{da_key} = distarray.local.empty(' 'distarray.local.maps.Distribution.from_shape({shape_name}, ' '{dist_name}, {grid_shape_name}, {comm_name}), {dtype_name})') ctx._execute(cmd.format(**locals())) - self.mdmap = mdmap + self.mdmap = distribution self.key = da_key self._dtype = dtype From 9bccc10bcd10ec671cb116736366125c6fc7863c Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 12:57:52 -0500 Subject: [PATCH 15/19] _make_mdmap_from_local_dimdata -> ...distribution... --- distarray/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distarray/client.py b/distarray/client.py index 066003ce..01a848ba 100644 --- a/distarray/client.py +++ b/distarray/client.py @@ -71,7 +71,7 @@ def is_LocalArray(typestring): {dim_data_name} = {local_name}.dim_data """ -def _make_mdmap_from_local_dimdata(local_name, context): +def _make_distribution_from_local_dimdata(local_name, context): dim_data_name = context._generate_key() context._execute(_DIMDATAS.format(local_name=local_name, dim_data_name=dim_data_name)) dim_datas = context._pull(dim_data_name) @@ -116,7 +116,7 @@ def from_localarrays(cls, key, context): """ da = cls.__new__(cls) da.key = key - da.mdmap = _make_mdmap_from_local_dimdata(key, context) + da.mdmap = _make_distribution_from_local_dimdata(key, context) da._dtype = _get_attribute(context, key, 'dtype') return da From 749ead6df6ec05d3b9495b6d824a70d4c44c80a8 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 13:00:04 -0500 Subject: [PATCH 16/19] mdmap -> distribution (DistArray attribute) --- distarray/client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/distarray/client.py b/distarray/client.py index 01a848ba..e8cd3a20 100644 --- a/distarray/client.py +++ b/distarray/client.py @@ -103,7 +103,7 @@ def __init__(self, distribution, dtype): 'distarray.local.maps.Distribution.from_shape({shape_name}, ' '{dist_name}, {grid_shape_name}, {comm_name}), {dtype_name})') ctx._execute(cmd.format(**locals())) - self.mdmap = distribution + self.distribution = distribution self.key = da_key self._dtype = dtype @@ -116,7 +116,7 @@ def from_localarrays(cls, key, context): """ da = cls.__new__(cls) da.key = key - da.mdmap = _make_distribution_from_local_dimdata(key, context) + da.distribution = _make_distribution_from_local_dimdata(key, context) da._dtype = _get_attribute(context, key, 'dtype') return da @@ -138,7 +138,7 @@ def __getitem__(self, index): return self.__getitem__(tuple_index) elif isinstance(index, tuple): - targets = self.mdmap.owning_targets(index) + targets = self.distribution.owning_targets(index) result_key = self.context._generate_key() fmt = '%s = %s.checked_getitem(%s)' statement = fmt % (result_key, self.key, index) @@ -163,7 +163,7 @@ def __setitem__(self, index, value): return self.__setitem__(tuple_index, value) elif isinstance(index, tuple): - targets = self.mdmap.owning_targets(index) + targets = self.distribution.owning_targets(index) result_key = self.context._generate_key() fmt = '%s = %s.checked_setitem(%s, %s)' statement = fmt % (result_key, self.key, index, value) @@ -177,11 +177,11 @@ def __setitem__(self, index, value): @property def context(self): - return self.mdmap.context + return self.distribution.context @property def shape(self): - return self.mdmap.shape + return self.distribution.shape @property def global_size(self): @@ -189,11 +189,11 @@ def global_size(self): @property def dist(self): - return self.mdmap.dist + return self.distribution.dist @property def grid_shape(self): - return self.mdmap.grid_shape + return self.distribution.grid_shape @property def ndim(self): From 46b7a9f748a8c179fe44dc8b6a2efc2150566a25 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 13:02:17 -0500 Subject: [PATCH 17/19] mdmap -> distribution (local usage in Context) --- distarray/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distarray/context.py b/distarray/context.py index ae887a0d..335c1bc2 100644 --- a/distarray/context.py +++ b/distarray/context.py @@ -320,8 +320,8 @@ def from_global_dim_data(self, global_dim_data, dtype=float): """ # global_dim_data is a sequence of dictionaries, one per dimension. - mdmap = Distribution(self, global_dim_data) - dim_data_per_rank = mdmap.get_local_dim_datas() + distribution = Distribution(self, global_dim_data) + dim_data_per_rank = distribution.get_local_dim_datas() if len(self.targets) != len(dim_data_per_rank): errmsg = "`dim_data_per_rank` must contain a dim_data for every rank." From 0ec0276e9a71d32436ee739ce81bf869d24642c0 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 13:03:11 -0500 Subject: [PATCH 18/19] mdmap -> distribution (local usage in a test) --- distarray/tests/test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distarray/tests/test_client.py b/distarray/tests/test_client.py index 0467024f..92308ec0 100644 --- a/distarray/tests/test_client.py +++ b/distarray/tests/test_client.py @@ -215,8 +215,8 @@ def test_global_dim_data_local_dim_data_equivalence(self): 'size': cols, }, ) - mdmap = Distribution(self.context, glb_dim_data) - actual = mdmap.get_local_dim_datas() + distribution = Distribution(self.context, glb_dim_data) + actual = distribution.get_local_dim_datas() expected = [ ({'block_size': 2, From b30d3ddbccfabbc25628470f98f1b529acc72b14 Mon Sep 17 00:00:00 2001 From: Robert David Grant Date: Tue, 29 Apr 2014 13:08:00 -0500 Subject: [PATCH 19/19] mdmap -> distribution (local usage in a test) --- distarray/tests/test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distarray/tests/test_client.py b/distarray/tests/test_client.py index 92308ec0..d434a113 100644 --- a/distarray/tests/test_client.py +++ b/distarray/tests/test_client.py @@ -300,8 +300,8 @@ def tearDown(self): def test___init__(self): shape = (100, 100) - mdmap = Distribution.from_shape(self.context, shape, ('b', 'c')) - da = DistArray(mdmap, dtype=int) + distribution = Distribution.from_shape(self.context, shape, ('b', 'c')) + da = DistArray(distribution, dtype=int) da.fill(42) nda = numpy.empty(shape, dtype=int) nda.fill(42)