diff --git a/distarray/dist/distarray.py b/distarray/dist/distarray.py index e0aceac0..f3379230 100644 --- a/distarray/dist/distarray.py +++ b/distarray/dist/distarray.py @@ -17,10 +17,12 @@ import operator from itertools import product from functools import reduce +from collections import Sequence import numpy as np import distarray +from distarray.metadata_utils import sanitize_indices from distarray.dist.maps import Distribution from distarray.utils import _raise_nie from distarray.metadata_utils import normalize_reduction_axes @@ -32,7 +34,6 @@ # Code # --------------------------------------------------------------------------- - class DistArray(object): __array_priority__ = 20.0 @@ -84,7 +85,8 @@ def get_dim_datas_and_dtype(arr): # has context, get dist and dtype elif (distribution is None) and (dtype is None): - res = context.apply(get_dim_datas_and_dtype, args=(key,)) + res = context.apply(get_dim_datas_and_dtype, args=(key,), + targets=targets) dim_datas = [i[0] for i in res] dtypes = [i[1] for i in res] da._dtype = dtypes[0] @@ -95,7 +97,8 @@ def get_dim_datas_and_dtype(arr): # has context and dtype, get dist elif (distribution is None) and (dtype is not None): da._dtype = dtype - dim_datas = context.apply(getattr, args=(key, 'dim_data')) + dim_datas = context.apply(getattr, args=(key, 'dim_data'), + targets=targets) da.distribution = Distribution.from_dim_data_per_rank(context, dim_datas, targets) @@ -128,10 +131,31 @@ def __repr__(self): (self.shape, self.targets) return s + def _process_return_value(self, result, return_proxy, index, targets): + + if return_proxy: + # proxy returned as result of slice + # slicing shouldn't alter the dtype + result = result[0] + return DistArray.from_localarrays(key=result, + context=self.context, + targets=targets, + dtype=self.dtype) + + elif isinstance(result, Sequence): + somethings = [i for i in result if i is not None] + if len(somethings) == 0: + # using checked_getitem and all return None + raise IndexError("Index %r is is not present." % (index,)) + if len(somethings) == 1: + return somethings[0] + else: + return result + else: + assert False # impossible is nothing + + def __getitem__(self, index): - #TODO: FIXME: major performance improvements possible here, - # especially for special cases like `index == slice(None)`. - # This would dramatically improve tondarray's performance. # to be run locally def checked_getitem(arr, index): @@ -141,38 +165,35 @@ def checked_getitem(arr, index): def raw_getitem(arr, index): return arr.global_index[index] - if isinstance(index, int) or isinstance(index, slice): - tuple_index = (index,) - return self.__getitem__(tuple_index) - - elif isinstance(index, tuple): - targets = self.distribution.owning_targets(index) - - args = (self.key, index) - if self.distribution.has_precise_index: - result = self.context.apply(raw_getitem, args=args, - targets=targets) - else: - result = self.context.apply(checked_getitem, args=args, - targets=targets) - result = [i for i in result if i is not None] - if len(result) != 1: - raise IndexError("Getting more than one result (%s) is not " - "supported yet." % (result,)) - elif result is None: - raise IndexError("Index %r is out of bounds" % (index,)) - else: - return result[0] - else: - raise TypeError("Invalid index type.") + # to be run locally + def get_slice(arr, index, ddpr, comm): + from distarray.local.maps import Distribution + local_distribution = Distribution(comm=comm, + dim_data=ddpr[comm.Get_rank()]) + result = arr.global_index.get_slice(index, local_distribution) + return proxyize(result) + + return_type, index = sanitize_indices(index, ndim=self.ndim, + shape=self.shape) + return_proxy = (return_type == 'view') + targets = self.distribution.owning_targets(index) + + args = [self.key, index] + if self.distribution.has_precise_index: + if return_proxy: # returning a new DistArray view + new_distribution = self.distribution.slice(index) + ddpr = new_distribution.get_dim_data_per_rank() + args.extend([ddpr, new_distribution.comm]) + local_fn = get_slice + else: # returning a value + local_fn = raw_getitem + else: # returning a value from unstructured + local_fn = checked_getitem + + result = self.context.apply(local_fn, args=args, targets=targets) + return self._process_return_value(result, return_proxy, index, targets) def __setitem__(self, index, value): - #TODO: FIXME: major performance improvements possible here. - # Especially when `index == slice(None)` and value is an - # ndarray, since for block and cyclic, we can generate slices of - # `value` and assign to local arrays. This would dramatically - # improve the fromndarray method's performance. - # to be run locally def checked_setitem(arr, index, value): return arr.global_index.checked_setitem(index, value) @@ -181,26 +202,57 @@ def checked_setitem(arr, index, value): def raw_setitem(arr, index, value): arr.global_index[index] = value - if isinstance(index, int) or isinstance(index, slice): - tuple_index = (index,) - return self.__setitem__(tuple_index, value) - - elif isinstance(index, tuple): - targets = self.distribution.owning_targets(index) - args = (self.key, index, value) - if self.distribution.has_precise_index: - self.context.apply(raw_setitem, args=args, targets=targets) + # to be run locally + def set_slice(arr, index, value, value_slices): + local_slice = value_slices[arr.comm_rank] + arr.global_index[index] = value[local_slice] + + set_type, index = sanitize_indices(index, ndim=self.ndim, + shape=self.shape) + + targets = self.distribution.owning_targets(index) + args = [self.key, index, value] + if self.distribution.has_precise_index: + if set_type == 'value': + local_fn = raw_setitem + elif set_type == 'view': + args[-1] = np.asarray(args[-1]) # convert to array + # this could be made more efficient + # we only need the bounds computed by distribution.slice + new_distribution = self.distribution.slice(index) + if args[-1].shape != new_distribution.shape: + msg = "Slice shape does not equal rvalue shape." + raise ValueError(msg) + ddpr = new_distribution.get_dim_data_per_rank() + def bounds_slice(dd): + if dd['dist_type'] == 'b': + return slice(dd['start'], dd['stop']) + elif dd['dist_type'] == 'n': + return slice(0, dd['size']) + else: + msg = "Function only works for 'n' and 'b' 'dist_type's" + raise TypeError(msg) + value_slices = [tuple(bounds_slice(dd) for dd in dim_data) + for dim_data in ddpr] + # but we need a data structure indexable by a target's rank + # assume contigious range of targets here + value_slices_per_target = [None] * len(self.targets) + value_slices_per_target[targets[0]:targets[-1]] = value_slices + args.append(value_slices_per_target) + local_fn = set_slice else: - result = self.context.apply(checked_setitem, args=args, - targets=targets) - result = [i for i in result if i is not None] - if len(result) > 1: - raise IndexError("Setting more than one result (%s) is " - "not supported yet." % (result,)) - elif result == []: - raise IndexError("Index %s is out of bounds" % (index,)) - else: - raise TypeError("Invalid index type.") + assert False + self.context.apply(local_fn, args=args, targets=targets) + + else: # setting unstructured elements + local_fn = checked_setitem + result = self.context.apply(local_fn, args=args, targets=targets) + result = [i for i in result if i is not None] + if len(result) > 1: + raise IndexError("Setting more than one result (%s) is " + "not supported yet." % (result,)) + elif result == []: + raise IndexError("Index %s is out of bounds" % (index,)) @property def context(self): @@ -246,7 +298,8 @@ def tondarray(self): """Returns the distributed array as an ndarray.""" arr = np.empty(self.shape, dtype=self.dtype) local_name = self.context._generate_key() - self.context._execute('%s = %s.copy()' % (local_name, self.key), targets=self.targets) + self.context._execute('%s = %s.copy()' % (local_name, self.key), + targets=self.targets) local_arrays = self.context._pull(local_name, targets=self.targets) for local_array in local_arrays: maps = (list(ax_map.global_iter) for ax_map in diff --git a/distarray/dist/maps.py b/distarray/dist/maps.py index bca5f2f6..bb33b980 100644 --- a/distarray/dist/maps.py +++ b/distarray/dist/maps.py @@ -21,11 +21,12 @@ `UnstructuredMap`. """ -from __future__ import absolute_import +from __future__ import division, absolute_import import operator from itertools import product from abc import ABCMeta, abstractmethod +from numbers import Integral import numpy as np @@ -34,11 +35,12 @@ from distarray.utils import remove_elements from distarray.metadata_utils import (normalize_dist, normalize_grid_shape, + normalize_dim_dict, + normalize_reduction_axes, make_grid_shape, - positivify, + sanitize_indices, _start_stop_block, - normalize_dim_dict, - normalize_reduction_axes) + tuple_intersection) def _dedup_dim_dicts(dim_dicts): @@ -191,9 +193,21 @@ def __init__(self, size, grid_size): msg = "grid_size for NoDistMap must be 1 (given %s)" raise ValueError(msg % grid_size) self.size = size + self.grid_size = grid_size def owners(self, idx): - return [0] if 0 <= idx < self.size else [] + if isinstance(idx, Integral): + return [0] if 0 <= idx < self.size else [] + elif isinstance(idx, slice): + start = idx.start if idx.start is not None else 0 + stop = idx.stop if idx.stop is not None else self.size + step = idx.step if idx.step is not None else 1 + if tuple_intersection((start, stop, step), (0, self.size)): + return [0] + else: + return [] + else: + raise TypeError("Index must be Integral or slice.") def get_dimdicts(self): return ({ @@ -203,6 +217,21 @@ def get_dimdicts(self): 'proc_grid_rank': 0, },) + def slice(self, idx): + """Make a new Map from a slice.""" + start = idx.start if idx.start is not None else 0 + stop = idx.stop if idx.stop is not None else self.size + step = idx.step if idx.step is not None else 1 + isection = tuple_intersection((start, stop, step), (0, self.size)) + if isection: + step = idx.step if idx.step is not None else 1 + isection_size = int(np.ceil((isection[1] - isection[0]) / step)) + else: + isection_size = 0 + + return {'dist_type': self.dist, + 'size': isection_size} + class BlockMap(MapBase): @@ -254,10 +283,21 @@ def __init__(self, size, grid_size): def owners(self, idx): coords = [] - for (coord, (lower, upper)) in enumerate(self.bounds): - if lower <= idx < upper: - coords.append(coord) - return coords + if isinstance(idx, Integral): + for (coord, (lower, upper)) in enumerate(self.bounds): + if lower <= idx < upper: + coords.append(coord) + return coords + elif isinstance(idx, slice): + start = idx.start if idx.start is not None else 0 + stop = idx.stop if idx.stop is not None else self.size + step = idx.step if idx.step is not None else 1 + for (coord, (lower, upper)) in enumerate(self.bounds): + if tuple_intersection((start, stop, step), (lower, upper)): + coords.append(coord) + return coords + else: + raise TypeError("Index must be Integral or slice.") def get_dimdicts(self): grid_ranks = range(len(self.bounds)) @@ -280,6 +320,25 @@ def get_dimdicts(self): }) return tuple(out) + def slice(self, idx): + """Make a new Map from a slice.""" + new_bounds = [0] + start = idx.start if idx.start is not None else 0 + step = idx.step if idx.step is not None else 1 + # iterate over the processes in this dimension + for proc_start, proc_stop in self.bounds: + stop = idx.stop if idx.stop is not None else proc_stop + isection = tuple_intersection((start, stop, step), + (proc_start, proc_stop)) + if isection: + isection_size = int(np.ceil((isection[1] - (isection[0])) / step)) + new_bounds.append(isection_size + new_bounds[-1]) + if len(new_bounds) == [0]: + new_bounds = [] + + return {'dist_type': self.dist, + 'bounds': new_bounds} + class BlockCyclicMap(MapBase): @@ -316,8 +375,12 @@ def __init__(self, size, grid_size, block_size=1): self.block_size = block_size def owners(self, idx): - idx_block = idx // self.block_size - return [idx_block % self.grid_size] + if isinstance(idx, Integral): + idx_block = idx // self.block_size + return [idx_block % self.grid_size] + else: + msg = "Index for BlockCyclicMap must be an Integral." + raise NotImplementedError(msg) def get_dimdicts(self): return tuple(({'dist_type': 'c', @@ -371,7 +434,11 @@ def owners(self, idx): # TODO: FIXME: for now, the unstructured map just returns all # processes. Can be optimized if we know the upper and lower bounds # for each local array's global indices. - return self._owners + if isinstance(idx, Integral): + return self._owners + else: + msg = "Index for UnstructuredMap must be an Integral." + raise NotImplementedError(msg) def get_dimdicts(self): if self.indices is None: @@ -595,6 +662,24 @@ def has_precise_index(self): """ return not any(isinstance(m, UnstructuredMap) for m in self.maps) + def slice(self, index_tuple): + """Make a new Distribution from a slice.""" + new_targets = self.owning_targets(index_tuple) + global_dim_data = [] + # iterate over the dimensions + for map_, idx in zip(self.maps, index_tuple): + if isinstance(idx, Integral): + continue # integral indexing returns reduced dimensionality + elif isinstance(idx, slice): + global_dim_data.append(map_.slice(idx)) + else: + msg = "Index must be a sequence of Integrals and slices." + raise TypeError(msg) + + return self.__class__(context=self.context, + global_dim_data=global_dim_data, + targets=new_targets) + def owning_ranks(self, idxs): """ Returns a list of ranks that may *possibly* own the location in the `idxs` tuple. @@ -606,7 +691,7 @@ def owning_ranks(self, idxs): If the `idxs` tuple is out of bounds, raises `IndexError`. """ - idxs = map(positivify, idxs, self.shape) # positivify and check + _, idxs = sanitize_indices(idxs, ndim=self.ndim, shape=self.shape) dim_coord_hits = [m.owners(idx) for (m, idx) in zip(self.maps, idxs)] all_coords = product(*dim_coord_hits) ranks = [self.rank_from_coords[c] for c in all_coords] diff --git a/distarray/dist/tests/test_distarray.py b/distarray/dist/tests/test_distarray.py index e276b3b7..479f8798 100644 --- a/distarray/dist/tests/test_distarray.py +++ b/distarray/dist/tests/test_distarray.py @@ -114,6 +114,218 @@ def test_global_tolocal_bug(self): numpy.testing.assert_array_equal(dap.tondarray(), ndarr) +class TestGetItemSlicing(ContextTestCase): + + def test_full_slice_block_dist(self): + size = 10 + expected = numpy.random.randint(11, size=size) + arr = self.context.fromarray(expected) + assert_array_equal(arr[:].toarray(), expected) + + def test_partial_slice_block_dist(self): + size = 10 + expected = numpy.random.randint(10, size=size) + arr = self.context.fromarray(expected) + assert_array_equal(arr[0:2].toarray(), expected[0:2]) + + def test_slice_a_slice_block_dist_0(self): + size = 10 + expected = numpy.random.randint(10, size=size) + arr = self.context.fromarray(expected) + s0 = arr[:9] + s1 = s0[0:5] + s2 = s1[:2] + assert_array_equal(s2.toarray(), expected[:2]) + + def test_slice_a_slice_block_dist_1(self): + size = 10 + expected = numpy.random.randint(10, size=size) + arr = self.context.fromarray(expected) + s0 = arr[:9] + s1 = s0[0:5] + s2 = s1[-2:] + assert_array_equal(s2.toarray(), expected[3:5]) + + def test_slice_block_dist_1d_with_step(self): + size = 10 + step = 2 + expected = numpy.random.randint(10, size=size) + darr = self.context.fromarray(expected) + assert_array_equal(darr[::step].toarray(), expected[::step]) + + def test_partial_slice_block_dist_2d(self): + shape = (10, 20) + expected = numpy.random.randint(10, size=shape) + arr = self.context.fromarray(expected) + assert_array_equal(arr[2:6, 3:10].toarray(), expected[2:6, 3:10]) + + def test_partial_negative_slice_block_dist_2d(self): + shape = (10, 20) + expected = numpy.random.randint(10, size=shape) + arr = self.context.fromarray(expected) + assert_array_equal(arr[-6:-2, -10:-3].toarray(), + expected[-6:-2, -10:-3]) + + def test_incomplete_slice_block_dist_2d(self): + shape = (10, 20) + expected = numpy.random.randint(10, size=shape) + arr = self.context.fromarray(expected) + assert_array_equal(arr[3:9].toarray(), expected[3:9]) + + def test_incomplete_index_block_dist_2d(self): + shape = (10, 20) + expected = numpy.random.randint(10, size=shape) + arr = self.context.fromarray(expected) + assert_array_equal(arr[1].toarray(), expected[1]) + + def test_trailing_ellipsis(self): + shape = (2, 3, 7, 6) + expected = numpy.random.randint(10, size=shape) + arr = self.context.fromarray(expected) + assert_array_equal(arr[1, ...].toarray(), expected[1, ...]) + + def test_leading_ellipsis(self): + shape = (2, 3, 7, 6) + expected = numpy.random.randint(10, size=shape) + arr = self.context.fromarray(expected) + assert_array_equal(arr[..., 3].toarray(), expected[..., 3]) + + def test_multiple_ellipsis(self): + shape = (2, 4, 2, 4, 1, 5) + expected = numpy.random.randint(10, size=shape) + arr = self.context.fromarray(expected) + assert_array_equal(arr[..., 3, ..., 4].toarray(), + expected[..., 3, ..., 4]) + + def test_vestigial_ellipsis(self): + shape = (1, 2, 3) + expected = numpy.random.randint(10, size=shape) + arr = self.context.fromarray(expected) + assert_array_equal(arr[0, :, 0, ...].toarray(), + expected[0, :, 0, ...]) + + +class TestSetItemSlicing(ContextTestCase): + + def test_small_1d_slice(self): + source = numpy.random.randint(10, size=20) + new_data = numpy.random.randint(10, size=3) + slc = slice(1, 4) + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_large_1d_slice(self): + source = numpy.random.randint(10, size=20) + new_data = numpy.random.randint(10, size=10) + slc = slice(5, 15) + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_1d_slice_with_step(self): + source = numpy.random.randint(10, size=20) + new_data = numpy.random.randint(10, size=5) + slc = slice(7, 17, 2) + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_2d_slice_0(self): + # on process boundaries + source = numpy.random.randint(10, size=(10, 20)) + new_data = numpy.random.randint(10, size=(5, 10)) + slc = (slice(5, 10), slice(5, 15)) + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_2d_slice_with_step(self): + source = numpy.random.randint(10, size=(10, 20)) + new_data = numpy.random.randint(10, size=(2, 5)) + slc = (slice(5, 10, 3), slice(5, 15, 2)) + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_2d_slice_1(self): + # off process boundaries + source = numpy.random.randint(10, size=(10, 20)) + new_data = numpy.random.randint(10, size=(5, 10)) + slc = (slice(3, 8), slice(9, 19)) + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_full_3d_slice(self): + source = numpy.random.randint(10, size=(3, 4, 5)) + new_data = numpy.random.randint(10, size=(3, 4, 5)) + slc = (slice(None), slice(None), slice(None)) + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_full_3d_slice_ellipsis(self): + source = numpy.random.randint(10, size=(3, 4, 5)) + new_data = numpy.random.randint(10, size=(3, 4, 5)) + slc = Ellipsis + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_3d_slice_ellipsis_with_step(self): + source = numpy.random.randint(10, size=(5, 4, 5)) + new_data = numpy.random.randint(10, size=(5, 2, 5)) + slc = (Ellipsis, slice(None, None, 2), Ellipsis) + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_partial_indexing_0(self): + source = numpy.random.randint(10, size=(3, 4, 5)) + new_data = numpy.random.randint(10, size=(4, 5)) + slc = (1,) + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_partial_indexing_1(self): + source = numpy.random.randint(10, size=(3, 4, 5)) + new_data = numpy.random.randint(10, size=(3, 5)) + slc = (slice(None), 2) + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_non_array_data(self): + source = numpy.random.randint(10, size=(3, 4)) + new_data = [42, 42, 42, 42] + slc = (2,) + arr = self.context.fromarray(source) + source[slc] = new_data + arr[slc] = new_data + assert_array_equal(arr.toarray(), source) + + def test_valueerror(self): + source = numpy.random.randint(10, size=21) + new_data = numpy.random.randint(10, size=10) + slc = slice(15, None) + arr = self.context.fromarray(source) + with self.assertRaises(ValueError): + arr[slc] = new_data + + class TestDistArrayCreationFromGlobalDimData(ContextTestCase): def test_from_global_dim_data_irregular_block(self): @@ -365,7 +577,8 @@ class TestDistArrayCreationSubSet(ContextTestCase): def test_create_target_subset(self): shape = (100, 100) subtargets = self.context.targets[::2] - distribution = Distribution.from_shape(self.context, shape=shape, targets=subtargets) + distribution = Distribution.from_shape(self.context, shape=shape, + targets=subtargets) darr = self.context.ones(distribution) lss = darr.get_localshapes() self.assertEqual(len(lss), len(subtargets)) diff --git a/distarray/dist/tests/test_maps.py b/distarray/dist/tests/test_maps.py index bc45b2e8..fd963899 100644 --- a/distarray/dist/tests/test_maps.py +++ b/distarray/dist/tests/test_maps.py @@ -4,23 +4,25 @@ # Distributed under the terms of the BSD License. See COPYING.rst. # --------------------------------------------------------------------------- +from __future__ import division + import unittest from random import randrange from distarray.externals.six.moves import range from distarray.testing import ContextTestCase -from distarray.dist import maps as client_map +from distarray.dist import maps class TestClientMap(ContextTestCase): def test_2D_bn(self): nrows, ncols = 31, 53 - cm = client_map.Distribution.from_shape(self.context, - (nrows, ncols), - {0: 'b'}, - (4, 1)) + cm = maps.Distribution.from_shape(self.context, + (nrows, ncols), + {0: 'b'}, + (4, 1)) chunksize = (nrows // 4) + 1 for _ in range(100): r, c = randrange(nrows), randrange(ncols) @@ -30,9 +32,10 @@ def test_2D_bn(self): def test_2D_bb(self): nrows, ncols = 3, 5 nprocs_per_dim = 2 - cm = client_map.Distribution.from_shape( - self.context, (nrows, ncols), ('b', 'b'), - (nprocs_per_dim, nprocs_per_dim)) + cm = maps.Distribution.from_shape(self.context, + (nrows, ncols), + ('b', 'b'), + (nprocs_per_dim, nprocs_per_dim)) row_chunks = nrows // nprocs_per_dim + 1 col_chunks = ncols // nprocs_per_dim + 1 for r in range(nrows): @@ -44,25 +47,28 @@ def test_2D_bb(self): def test_2D_cc(self): nrows, ncols = 3, 5 nprocs_per_dim = 2 - cm = client_map.Distribution.from_shape( - self.context, (nrows, ncols), ('c', 'c'), - (nprocs_per_dim, nprocs_per_dim)) + cm = maps.Distribution.from_shape(self.context, + (nrows, ncols), + ('c', 'c'), + (nprocs_per_dim, nprocs_per_dim)) for r in range(nrows): for c in range(ncols): - rank = (r % nprocs_per_dim) * nprocs_per_dim + (c % nprocs_per_dim) + rank = ((r % nprocs_per_dim) * nprocs_per_dim + + (c % nprocs_per_dim)) actual = cm.owning_ranks((r,c)) self.assertSequenceEqual(actual, [rank]) - def test_is_compatible(self): nr, nc, nd = 10**5, 10**6, 10**4 - cm0 = client_map.Distribution.from_shape( - self.context, (nr, nc, nd), ('b', 'c', 'n')) + cm0 = maps.Distribution.from_shape(self.context, + (nr, nc, nd), + ('b', 'c', 'n')) self.assertTrue(cm0.is_compatible(cm0)) - cm1 = client_map.Distribution.from_shape( - self.context, (nr, nc, nd), ('b', 'c', 'n')) + cm1 = maps.Distribution.from_shape(self.context, + (nr, nc, nd), + ('b', 'c', 'n')) self.assertTrue(cm1.is_compatible(cm1)) self.assertTrue(cm0.is_compatible(cm1)) @@ -70,8 +76,9 @@ def test_is_compatible(self): nr -= 1; nc -= 1; nd -= 1 - cm2 = client_map.Distribution.from_shape( - self.context, (nr, nc, nd), ('b', 'c', 'n')) + cm2 = maps.Distribution.from_shape(self.context, + (nr, nc, nd), + ('b', 'c', 'n')) self.assertFalse(cm1.is_compatible(cm2)) self.assertFalse(cm2.is_compatible(cm1)) @@ -79,9 +86,10 @@ def test_is_compatible(self): def test_reduce(self): nr, nc, nd = 10**5, 10**6, 10**4 - dist = client_map.Distribution.from_shape( - self.context, (nr, nc, nd), ('b', 'c', 'n'), - grid_shape=(2, 2, 1)) + dist = maps.Distribution.from_shape(self.context, + (nr, nc, nd), + ('b', 'c', 'n'), + grid_shape=(2, 2, 1)) new_dist0 = dist.reduce(axes=[0]) self.assertEqual(new_dist0.dist, ('c', 'n')) @@ -92,7 +100,8 @@ def test_reduce(self): new_dist1 = dist.reduce(axes=[1]) self.assertEqual(new_dist1.dist, ('b', 'n')) self.assertSequenceEqual(new_dist1.shape, (nr, nd)) - self.assertEqual(new_dist1.grid_shape, dist.grid_shape[:1]+dist.grid_shape[2:]) + self.assertEqual(new_dist1.grid_shape, + dist.grid_shape[:1] + dist.grid_shape[2:]) self.assertLess(set(new_dist1.targets), set(dist.targets)) new_dist2 = dist.reduce(axes=[2]) @@ -103,7 +112,7 @@ def test_reduce(self): def test_reduce_0D(self): N = 10**5 - dist = client_map.Distribution.from_shape(self.context, (N,)) + dist = maps.Distribution.from_shape(self.context, (N,)) new_dist = dist.reduce(axes=[0]) self.assertEqual(new_dist.dist, ()) self.assertSequenceEqual(new_dist.shape, ()) @@ -111,11 +120,93 @@ def test_reduce_0D(self): self.assertEqual(set(new_dist.targets), set(dist.targets[:1])) +class TestSlice(ContextTestCase): + + def test_from_partial_slice_1d(self): + d0 = maps.Distribution.from_shape(context=self.context, shape=(15,)) + + s = (slice(0, 3),) + d1 = d0.slice(s) + + self.assertEqual(len(d0.maps), len(d1.maps)) + self.assertSequenceEqual(d1.dist, d0.dist) + self.assertSequenceEqual(d1.targets, [0]) + self.assertSequenceEqual(d1.shape, (3,)) + + def test_from_full_slice_1d(self): + d0 = maps.Distribution.from_shape(context=self.context, shape=(15,)) + + s = (slice(None),) + d1 = d0.slice(s) + + self.assertEqual(len(d0.maps), len(d1.maps)) + self.assertSequenceEqual(d1.dist, d0.dist) + self.assertSequenceEqual(d1.targets, d0.targets) + self.assertSequenceEqual(d1.maps[0].bounds, d0.maps[0].bounds) + + def test_from_full_slice_with_step_1d_0(self): + d0 = maps.Distribution.from_shape(context=self.context, shape=(15,)) + + s = (slice(None, None, 2),) + d1 = d0.slice(s) + + self.assertEqual(len(d0.maps), len(d1.maps)) + self.assertSequenceEqual(d1.dist, d0.dist) + self.assertSequenceEqual(d1.targets, d0.targets) + self.assertEqual(d1.maps[0].bounds[0][0], d0.maps[0].bounds[0][0]) + + def test_from_full_slice_with_step_1d_1(self): + d0 = maps.Distribution.from_shape(context=self.context, shape=(30,)) + step = 4 + + s = (slice(4, None, step),) + d1 = d0.slice(s) + + self.assertEqual(len(d0.maps), len(d1.maps)) + self.assertSequenceEqual(d1.dist, d0.dist) + self.assertSequenceEqual(d1.targets, d0.targets) + self.assertEqual(d1.maps[0].bounds[0][0], d0.maps[0].bounds[0][0]) + + def test_from_full_slice_2d(self): + d0 = maps.Distribution.from_shape(context=self.context, shape=(15, 20)) + + s = (slice(None), slice(None)) + d1 = d0.slice(s) + + self.assertEqual(len(d0.maps), len(d1.maps)) + self.assertSequenceEqual(d1.dist, d0.dist) + for m0, m1 in zip(d0.maps, d1.maps): + if m0.dist == 'b': + self.assertSequenceEqual(m0.bounds, m1.bounds) + self.assertSequenceEqual(d1.targets, d0.targets) + + def test_from_partial_slice_2d(self): + d0 = maps.Distribution.from_shape(context=self.context, shape=(15, 20)) + + s = (slice(3, 7), 4) + d1 = d0.slice(s) + + self.assertEqual(len(d0.maps)-1, len(d1.maps)) + self.assertSequenceEqual(d1.dist, d0.dist[:-1]) + for m, expected in zip(d1.maps, ([(0, 1), (1, 4)], [(0, 1)])): + self.assertSequenceEqual(m.bounds, expected) + + def test_full_slice_with_int_2d(self): + d0 = maps.Distribution.from_shape(context=self.context, shape=(15, 20)) + + s = (slice(None), 4) + d1 = d0.slice(s) + + self.assertEqual(len(d0.maps)-1, len(d1.maps)) + self.assertSequenceEqual(d1.dist, d0.dist[:-1]) + self.assertEqual(d1.shape, (15,)) + + class TestDistributionCreation(ContextTestCase): def test_all_n_dist(self): - distribution = client_map.Distribution.from_shape(self.context, - shape=(3, 3), - dist=('n', 'n')) + distribution = maps.Distribution.from_shape(self.context, + shape=(3, 3), + dist=('n', 'n')) self.context.ones(distribution) diff --git a/distarray/local/localarray.py b/distarray/local/localarray.py index b2fe829e..a8168ed5 100644 --- a/distarray/local/localarray.py +++ b/distarray/local/localarray.py @@ -11,37 +11,22 @@ # Imports # --------------------------------------------------------------------------- from collections import Mapping -from numbers import Integral import numpy as np from distarray.externals import six from distarray.externals.six.moves import zip -from distarray.local.mpiutils import MPI from distarray.utils import _raise_nie +from distarray.metadata_utils import sanitize_indices + +from distarray.local.mpiutils import MPI from distarray.local import format, maps from distarray.local.error import InvalidDimensionError, IncompatibleArrayError -# Register numpy integer types with numbers.Integral ABC. -Integral.register(np.signedinteger) -Integral.register(np.unsignedinteger) - - -def _sanitize_indices(indices): - if isinstance(indices, Integral) or isinstance(indices, slice): - return (indices,) - elif all(isinstance(i, Integral) or isinstance(i, slice) for i in indices): - return indices - else: - raise TypeError("Index must be a sequence of ints and slices") - - class GlobalIndex(object): - """Object which provides access to global indexing on - LocalArrays. - """ + """Object which provides access to global indexing on LocalArrays.""" def __init__(self, distribution, ndarray): self.distribution = distribution self.ndarray = ndarray @@ -65,16 +50,35 @@ def global_to_local(self, *global_ind): def local_to_global(self, *local_ind): return self.distribution.global_from_local(*local_ind) + def get_slice(self, global_inds, new_distribution): + try: + local_inds = self.global_to_local(*global_inds) + except KeyError as err: + raise IndexError(err) + view = self.ndarray[local_inds] + return LocalArray(distribution=new_distribution, + dtype=self.ndarray.dtype, + buf=view) + def __getitem__(self, global_inds): - global_inds = _sanitize_indices(global_inds) + return_type, global_inds = sanitize_indices(global_inds) try: local_inds = self.global_to_local(*global_inds) - return self.ndarray[local_inds] except KeyError as err: raise IndexError(err) + ndarray_view = self.ndarray[local_inds] + + if return_type == 'value': + return ndarray_view + elif return_type == 'view': + msg = "__getitem__ does not support slices. See `get_slice`." + raise TypeError(msg) + else: + assert False # impossible is nothing + def __setitem__(self, global_inds, value): - global_inds = _sanitize_indices(global_inds) + _, global_inds = sanitize_indices(global_inds) try: local_inds = self.global_to_local(*global_inds) self.ndarray[local_inds] = value @@ -399,7 +403,14 @@ def __len__(self): def __getitem__(self, index): """Get a local item.""" - return self.ndarray[index] + return_type, index = sanitize_indices(index) + if return_type == 'value': + return self.ndarray[index] + elif return_type == 'view': + msg = "__getitem__ does not support slices. See `global_index.get_item`." + raise TypeError(msg) + else: + assert False # impossible is nothing def __setitem__(self, index, value): """Set a local item.""" diff --git a/distarray/local/maps.py b/distarray/local/maps.py index faa7ade7..9e9a1484 100644 --- a/distarray/local/maps.py +++ b/distarray/local/maps.py @@ -22,6 +22,7 @@ import operator from functools import reduce +from numbers import Integral import numpy as np from distarray.externals.six.moves import range, zip @@ -29,7 +30,11 @@ from distarray.local import construct from distarray.metadata_utils import (make_grid_shape, normalize_grid_shape, normalize_dist, distribute_indices, - positivify) + sanitize_indices) + +# Register numpy integer types with numbers.Integral ABC. +Integral.register(np.signedinteger) +Integral.register(np.unsignedinteger) class Distribution(object): @@ -134,7 +139,7 @@ def rank_from_coords(self, coords): def local_from_global(self, *global_ind): """ Given `global_ind` indices, translate into local indices.""" - global_ind = tuple(map(positivify, global_ind, self.global_shape)) + _, idxs = sanitize_indices(global_ind, self.ndim, self.global_shape) return tuple(self._maps[dim].local_from_global(global_ind[dim]) for dim in range(self.ndim)) @@ -200,14 +205,40 @@ def __init__(self, global_size, grid_size, grid_rank, start, stop): self.grid_rank = grid_rank def local_from_global(self, gidx): - if gidx < self.start or gidx >= self.stop: - raise IndexError("Global index %s out of bounds" % gidx) - return gidx - self.start + if isinstance(gidx, Integral): + if gidx < self.start or gidx >= self.stop: + raise IndexError("Global index %s out of bounds" % gidx) + return gidx - self.start + elif isinstance(gidx, slice): + # we don't make the effort to compute the exact slice + # `__getitem__` doesn't care about overly-large slices, we just + # have to get the offset from the start correct based on the `step` + start = gidx.start if gidx.start is not None else 0 + stop = gidx.stop if gidx.stop is not None else self.global_size + step = gidx.step if gidx.step is not None else 1 + new_start = start - self.start + if new_start < 0: # don't allow negative starts + new_start += step * abs(new_start // step) + if new_start < 0: + new_start += step + new_stop = stop - self.start + return slice(new_start, new_stop, gidx.step) + else: + raise TypeError("Index must be Integral or slice.") def global_from_local(self, lidx): - if lidx >= self.local_size: - raise IndexError("Local index %s out of bounds" % lidx) - return lidx + self.start + if isinstance(lidx, Integral): + if lidx >= self.local_size: + raise IndexError("Local index %s out of bounds" % lidx) + return lidx + self.start + elif isinstance(lidx, slice): + start = lidx.start if lidx.start is not None else 0 + stop = lidx.stop if lidx.stop is not None else self.global_size + new_start = start + self.start + new_stop = stop + self.start + return slice(new_start, new_stop) + else: + raise TypeError("Index must be Integral or slice.") @property def dim_dict(self): @@ -248,16 +279,21 @@ def __init__(self, global_size, grid_size, grid_rank, start): self.local_size = (global_size - 1 - grid_rank) // grid_size + 1 self.global_size = global_size - def local_from_global(self, gidx): - if (gidx - self.start) % self.grid_size: - raise IndexError("Global index %s out of bounds" % gidx) - return (gidx - self.start) // self.grid_size + if isinstance(gidx, Integral): + if (gidx - self.start) % self.grid_size: + raise IndexError("Global index %s out of bounds" % gidx) + return (gidx - self.start) // self.grid_size + else: + raise NotImplementedError("Index must be Integral.") def global_from_local(self, lidx): - if lidx >= self.local_size: - raise IndexError("Local index %s out of bounds" % lidx) - return (lidx * self.grid_size) + self.start + if isinstance(lidx, Integral): + if lidx >= self.local_size: + raise IndexError("Local index %s out of bounds" % lidx) + return (lidx * self.grid_size) + self.start + else: + raise NotImplementedError("Index must be Integral.") @property def dim_dict(self): @@ -299,19 +335,24 @@ def __init__(self, global_size, grid_size, grid_rank, start, block_size): self.local_size = local_nblocks * block_size + local_partial self.global_size = global_size - def local_from_global(self, gidx): - global_block, offset = divmod(gidx, self.block_size) - if (global_block - self.start_block) % self.grid_size: - raise IndexError("Global index %s out of bounds" % gidx) - return self.block_size * ((global_block - self.start_block) // self.grid_size) + offset + if isinstance(gidx, Integral): + global_block, offset = divmod(gidx, self.block_size) + if (global_block - self.start_block) % self.grid_size: + raise IndexError("Global index %s out of bounds" % gidx) + return self.block_size * ((global_block - self.start_block) // self.grid_size) + offset + else: + raise NotImplementedError("Index must be Integral.") def global_from_local(self, lidx): - if lidx >= self.local_size: - raise IndexError("Local index %s out of bounds" % lidx) - local_block, offset = divmod(lidx, self.block_size) - global_block = (local_block * self.grid_size) + self.start_block - return global_block * self.block_size + offset + if isinstance(lidx, Integral): + if lidx >= self.local_size: + raise IndexError("Local index %s out of bounds" % lidx) + local_block, offset = divmod(lidx, self.block_size) + global_block = (local_block * self.grid_size) + self.start_block + return global_block * self.block_size + offset + else: + raise NotImplementedError("Index must be Integral.") @property def dim_dict(self): @@ -352,14 +393,20 @@ def __init__(self, global_size, grid_size, grid_rank, indices): self._local_index = dict(zip(self.indices, local_indices)) def local_from_global(self, gidx): - try: - lidx = self._local_index[gidx] - except KeyError: - raise IndexError("Global index %s out of bounds" % gidx) - return lidx + if isinstance(gidx, Integral): + try: + lidx = self._local_index[gidx] + except KeyError: + raise IndexError("Global index %s out of bounds" % gidx) + return lidx + else: + raise NotImplementedError("Index must be Integral.") def global_from_local(self, lidx): - return self.indices[lidx] + if isinstance(lidx, Integral): + return self.indices[lidx] + else: + raise NotImplementedError("Index must be Integral.") @property def dim_dict(self): diff --git a/distarray/local/tests/paralleltest_localarray.py b/distarray/local/tests/paralleltest_localarray.py index fc40869f..4d5a24da 100644 --- a/distarray/local/tests/paralleltest_localarray.py +++ b/distarray/local/tests/paralleltest_localarray.py @@ -7,11 +7,12 @@ import unittest import numpy as np +from numpy.testing import assert_array_equal from distarray import utils from distarray.testing import (MpiTestCase, assert_localarrays_allclose, assert_localarrays_equal) -from distarray.local.localarray import LocalArray, ndenumerate +from distarray.local.localarray import LocalArray, ndenumerate, ones from distarray.local.maps import Distribution from distarray.local.error import InvalidDimensionError, IncompatibleArrayError @@ -340,6 +341,44 @@ def test_pack_unpack_index(self): self.assertEqual(global_inds, a.unpack_index(packed_ind)) +class TestSlicing(MpiTestCase): + + comm_size = 2 + + def test_slicing(self): + distribution = Distribution.from_shape(self.comm, + (16, 16), + dist=('b', 'n')) + a = ones(distribution) + if self.comm.Get_rank() == 0: + dd00 = {"dist_type": 'b', + "size": 5, + "start": 0, + "stop": 3, + "proc_grid_size": 2, + "proc_grid_rank": 0} + dd01 = {"dist_type": 'n', + "size": 16} + + new_distribution = Distribution(self.comm, [dd00, dd01]) + rvals = a.global_index.get_slice((slice(5, None), slice(None)), + new_distribution=new_distribution) + assert_array_equal(rvals, np.ones((3, 16))) + + elif self.comm.Get_rank() == 1: + dd10 = {"dist_type": 'b', + "size": 5, + "start": 3, + "stop": 5, + "proc_grid_size": 2, + "proc_grid_rank": 1} + dd11 = {"dist_type": 'n', + "size": 16} + new_distribution = Distribution(self.comm, [dd10, dd11]) + rvals = a.global_index.get_slice((slice(None, 10), slice(None)), + new_distribution=new_distribution) + assert_array_equal(rvals, np.ones((2, 16))) + class TestLocalArrayMethods(MpiTestCase): ddpr = [ diff --git a/distarray/metadata_utils.py b/distarray/metadata_utils.py index fda53a25..c85aacfc 100644 --- a/distarray/metadata_utils.py +++ b/distarray/metadata_utils.py @@ -4,16 +4,24 @@ # Distributed under the terms of the BSD License. See COPYING.rst. # --------------------------------------------------------------------------- +from __future__ import division + import operator from itertools import product from functools import reduce +from numbers import Integral from collections import Sequence, Mapping import numpy from distarray import utils from distarray.externals.six import next -from distarray.externals.six.moves import map +from distarray.externals.six.moves import map, zip + + +# Register numpy integer types with numbers.Integral ABC. +Integral.register(numpy.signedinteger) +Integral.register(numpy.unsignedinteger) class InvalidGridShapeError(Exception): @@ -212,13 +220,153 @@ def normalize_dim_dict(dd): dd['proc_grid_rank'] = 0 -def positivify(index, size): - if 0 <= index < size: +def _positivify(index, size): + """Return a positive index offset from a Sequence's start.""" + if index is None or index >= 0: return index - elif -size <= index < 0: + elif index < 0: return size + index + +def _check_bounds(index, size): + """Check if an index is in bounds. + + Assumes a positive index as returned by _positivify. + """ + if not 0 <= index < size: + raise IndexError("Index %r out of bounds" % index) + + +def tuple_intersection(t0, t1): + """Compute intersection of a (start, stop, step) and a (start, stop) tuple. + + Assumes all values are positive. + + Parameters + ---------- + t0: 2-tuple or 3-tuple + Tuple of (start, stop, [step]) representing an index range + t1: 2-tuple + Tuple of (start, stop) representing an index range + + Returns + ------- + 3-tuple or None + A tightly bounded interval. + """ + if len(t0) == 2 or t0[2] is None: + # default step is 1 + t0 = (t0[0], t0[1], 1) + + start0, stop0, step0 = t0 + start1, stop1 = t1 + if start0 < start1: + n = int(numpy.ceil((start1 - start0) / step0)) + start2 = start0 + n*step0 + else: + start2 = start0 + + max_stop = min(t0[1], t1[1]) + if (max_stop - start2) % step0 == 0: + n = ((max_stop - start2) // step0) - 1 + else: + n = (max_stop - start2) // step0 + stop2 = (start2 + n*step0) + 1 + return (start2, stop2, step0) if stop2 > start2 else None + + +def positivify(index, size): + """Check that an index is within bounds and return a positive version. + + Parameters + ---------- + index : Integral or slice + size : Integral + + Raises + ------ + IndexError + for out-of-bounds indices + """ + if isinstance(index, Integral): + index = _positivify(index, size) + _check_bounds(index, size) + return index + elif isinstance(index, slice): + start = _positivify(index.start, size) + stop = _positivify(index.stop, size) + # slice indexing doesn't check bounds + return slice(start, stop, index.step) + else: + raise TypeError("`index` must be of type Integral or slice.") + + +def sanitize_indices(indices, ndim=None, shape=None): + """Classify and sanitize `indices`. + + * Wrap naked Integral, slice, or Ellipsis indices into tuples + * Classify result as 'value' or 'view' + * Expand `Ellipsis` objects to slices + * If the length of the tuple-ized `indices` is < ndim (and it's + provided), add slice(None)'s to indices until `indices` is ndim long + * If `shape` is provided, call `positivify` on the indices + + Raises + ------ + TypeError + If `indices` is other than Integral, slice or a Sequence of these + IndexError + If len(indices) > ndim + + Returns + ------- + 2-tuple of (str, n-tuple of slices and Integral values) + """ + if isinstance(indices, Integral): + rtype, sanitized = 'value', (indices,) + elif isinstance(indices, slice) or indices is Ellipsis: + rtype, sanitized = 'view', (indices,) + elif all(isinstance(i, Integral) for i in indices): + rtype, sanitized = 'value', indices + elif all(isinstance(i, Integral) + or isinstance(i, slice) + or i is Ellipsis for i in indices): + rtype, sanitized = 'view', indices else: - raise IndexError("Index %s out of bounds" % index) + msg = ("Index must be an Integral, a slice, or a sequence of " + "Integrals and slices.") + raise IndexError(msg) + + if Ellipsis in sanitized: + if ndim is None: + raise RuntimeError("Can't call `sanitize_indices` on Ellipsis " + "without providing `ndim`.") + # expand first Ellipsis + diff = ndim - (len(sanitized) - 1) + filler = (slice(None),) * diff + epos = sanitized.index(Ellipsis) + sanitized = sanitized[:epos] + filler + sanitized[epos+1:] + + # remaining Ellipsis objects are just converted to slices + def replace_ellipsis(idx): + if idx is Ellipsis: + return slice(None) + else: + return idx + sanitized = tuple(replace_ellipsis(i) for i in sanitized) + + if ndim is not None: + diff = ndim - len(sanitized) + if diff < 0: + raise IndexError("Too many indices.") + if diff > 0: + # allow incomplete indexing + rtype = 'view' + sanitized = sanitized + (slice(None),) * diff + + if shape is not None: + sanitized = tuple(positivify(i, size) for (i, size) in zip(sanitized, + shape)) + return (rtype, sanitized) def normalize_reduction_axes(axes, ndim): diff --git a/distarray/tests/test_metadata_utils.py b/distarray/tests/test_metadata_utils.py index bab24861..577a1c30 100644 --- a/distarray/tests/test_metadata_utils.py +++ b/distarray/tests/test_metadata_utils.py @@ -25,6 +25,194 @@ def test_negative_index(self): result = metadata_utils.positivify(-2, 10) self.assertEqual(result, 8) + def test_out_of_bounds_positive(self): + with self.assertRaises(IndexError): + metadata_utils.positivify(11, 10) + + def test_out_of_bounds_negative(self): + with self.assertRaises(IndexError): + metadata_utils.positivify(-51, 10) + + def test_positive_slice(self): + s = slice(5, 7) + result = metadata_utils.positivify(s, 10) + self.assertEqual(result, s) + + def test_negative_slice_stop(self): + s = slice(5, -2) + result = metadata_utils.positivify(s, 10) + expected = slice(5, 8) + self.assertEqual(result, expected) + + def test_no_slice_start(self): + s = slice(5) + result = metadata_utils.positivify(s, 10) + expected = s + self.assertEqual(result, expected) + + def test_no_slice_stop(self): + s = slice(5, None) + result = metadata_utils.positivify(s, 10) + expected = s + self.assertEqual(result, expected) + + def test_positive_slice_with_step(self): + s = slice(5, 7, 2) + result = metadata_utils.positivify(s, 10) + expected = s + self.assertEqual(result, expected) + + def test_negative_slice_with_step(self): + s = slice(-7, -1, 2) + result = metadata_utils.positivify(s, 10) + expected = slice(3, 9, 2) + self.assertEqual(result, expected) + + def test_out_of_bounds_slice(self): + s = slice(50, 90) + result = metadata_utils.positivify(s, 10) + self.assertEqual(result, s) + + +class TestSanitizeIndices(unittest.TestCase): + + def test_value_index(self): + tag, sanitized = metadata_utils.sanitize_indices(10) + self.assertSequenceEqual(sanitized, (10,)) + self.assertEqual(tag, 'value') + + def test_slice_index(self): + tag, sanitized = metadata_utils.sanitize_indices(slice(10, 20)) + self.assertSequenceEqual(sanitized, (slice(10, 20),)) + self.assertEqual(tag, 'view') + + def test_tuple_of_values(self): + tag, sanitized = metadata_utils.sanitize_indices((5, 10)) + self.assertSequenceEqual(sanitized, (5, 10)) + self.assertEqual(tag, 'value') + + def test_tuple_of_slices(self): + slices = slice(10, 20), slice(20, 30), slice(40, 50) + tag, sanitized = metadata_utils.sanitize_indices(slices) + self.assertSequenceEqual(sanitized, slices) + self.assertEqual(tag, 'view') + + def test_tuple_of_mixed(self): + slices = slice(10, 20), 25, slice(40, 50) + tag, sanitized = metadata_utils.sanitize_indices(slices) + self.assertSequenceEqual(sanitized, slices) + self.assertEqual(tag, 'view') + + def test_incomplete_indexing_values(self): + slices = 10, 20, 25, 40, 50 + tag, sanitized = metadata_utils.sanitize_indices(slices, ndim=10) + self.assertSequenceEqual(sanitized, slices + (slice(None),) * 5) + self.assertEqual(tag, 'view') + + def test_incomplete_indexing_mixed(self): + slices = slice(10, 20), 25, slice(40, 50) + tag, sanitized = metadata_utils.sanitize_indices(slices, ndim=10) + self.assertSequenceEqual(sanitized, slices + (slice(None),) * 7) + self.assertEqual(tag, 'view') + + def test_too_many_indices(self): + with self.assertRaises(IndexError): + metadata_utils.sanitize_indices((2, 3, 4), ndim=2) + + def test_trailing_ellipsis(self): + ndim = 5 + tag, sanitized = metadata_utils.sanitize_indices((10, Ellipsis), + ndim=ndim) + self.assertEqual(sanitized, (10,) + (slice(None),) * (ndim-1)) + + def test_leading_ellipsis(self): + ndim = 5 + tag, sanitized = metadata_utils.sanitize_indices((Ellipsis, 10), + ndim=ndim) + self.assertEqual(sanitized, (slice(None),) * (ndim-1) + (10,)) + + def test_multiple_ellipsis(self): + ndim = 6 + tag, sanitized = metadata_utils.sanitize_indices((Ellipsis, 10, + Ellipsis), + ndim=ndim) + self.assertEqual(sanitized, (slice(None),) * 4 + (10, slice(None))) + + def test_step(self): + # currently doesn't touch step + indices = (slice(None, None, 2), slice(None, 8, 4)) + tag, sanitized = metadata_utils.sanitize_indices(indices) + self.assertEqual(tag, 'view') + self.assertEqual(sanitized, indices) + + +class TestTupleIntersection(unittest.TestCase): + + def check_intersection_and_reverse(self, t0, t1, expected): + result = metadata_utils.tuple_intersection(t0, t1) + self.assertEqual(result, expected) + result = metadata_utils.tuple_intersection(t1, t0) + self.assertEqual(result, expected) + + def test_no_step_full_enclosure(self): + t0 = (0, 60) + t1 = (15, 30) + expected = (15, 30, 1) + self.check_intersection_and_reverse(t0, t1, expected) + + def test_no_step_partial_overlap(self): + t0 = (0, 60) + t1 = (15, 90) + expected = (15, 60, 1) + self.check_intersection_and_reverse(t0, t1, expected) + + def test_no_step_no_overlap(self): + t0 = (0, 60) + t1 = (80, 130) + expected = None + self.check_intersection_and_reverse(t0, t1, expected) + + def test_no_step_partial_overlap_0(self): + t0 = (0, 60) + t1 = (15, 90) + expected = (15, 60, 1) + self.check_intersection_and_reverse(t0, t1, expected) + + def test_no_step_partial_overlap_1(self): + # regression test + t0 = (0, 4) + t1 = (3, 7) + expected = (3, 4, 1) + self.check_intersection_and_reverse(t0, t1, expected) + + def test_with_step_1(self): + t0 = (0, 60, 1) + t1 = (15, 30) + expected = (15, 30, 1) + result = metadata_utils.tuple_intersection(t0, t1) + self.assertSequenceEqual(result, expected) + + def test_with_step_2(self): + t0 = (0, 60, 2) + t1 = (15, 30) + expected = (16, 29, 2) + result = metadata_utils.tuple_intersection(t0, t1) + self.assertSequenceEqual(result, expected) + + def test_with_step_3(self): + t0 = (0, 59, 2) + t1 = (15, 90) + expected = (16, 59, 2) + result = metadata_utils.tuple_intersection(t0, t1) + self.assertSequenceEqual(result, expected) + + def test_big_step(self): + t0 = (0, 59, 1000) + t1 = (15, 90) + expected = None + result = metadata_utils.tuple_intersection(t0, t1) + self.assertEqual(result, expected) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/distarray/tests/test_utils.py b/distarray/tests/test_utils.py index c6308699..7dd37b1b 100644 --- a/distarray/tests/test_utils.py +++ b/distarray/tests/test_utils.py @@ -31,24 +31,6 @@ def test_mult_partitions(self): self.assertEqual(utils.mult_partitions(6, 3), [(1, 1, 6), (1, 2, 3)]) -class TestSanitizeIndices(unittest.TestCase): - - def test_point(self): - itype, inds = utils.sanitize_indices(1) - self.assertEqual(itype, 'point') - self.assertEqual(inds, (1,)) - - def test_slice(self): - itype, inds = utils.sanitize_indices(slice(1,10)) - self.assertEqual(itype, 'view') - self.assertEqual(inds, (slice(1,10),)) - - def test_mixed(self): - provided = (5, 3, slice(7, 10, 2), 99, slice(1,10)) - itype, inds = utils.sanitize_indices(provided) - self.assertEqual(itype, 'view') - self.assertEqual(inds, provided) - class TestSliceIntersection(unittest.TestCase): @@ -105,5 +87,7 @@ def test_count_round_trips(self): view.execute('42') self.assertEqual(r.count, len(view)) + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/distarray/utils.py b/distarray/utils.py index c631fdaf..8bfa6c07 100644 --- a/distarray/utils.py +++ b/distarray/utils.py @@ -109,37 +109,6 @@ def _raise_nie(): raise NotImplementedError(msg) -def sanitize_indices(indices): - """Check and possibly sanitize indices. - - Parameters - ---------- - indices : int, slice, or sequence of ints and slices - If an int or slice is passed in, it is converted to a - 1-tuple. - - Returns - ------- - 2-tuple - ('point', indices) if all `indices` are ints, or - ('view', indices) if some `indices` are slices. - - Raises - ------ - TypeError - If `indices` is not all ints or slices. - """ - - if isinstance(indices, int) or isinstance(indices, slice): - return sanitize_indices((indices,)) - elif all(isinstance(i, int) for i in indices): - return 'point', indices - elif all(isinstance(i, int) or isinstance(i, slice) for i in indices): - return 'view', indices - else: - raise TypeError("Index must be a sequence of ints and slices") - - def slice_intersection(s1, s2): """Compute a slice that represents the intersection of two slices. diff --git a/docs/2014-06-apug/2014-06-apug.ipynb b/docs/2014-06-apug/2014-06-apug.ipynb new file mode 100644 index 00000000..e13c36c8 --- /dev/null +++ b/docs/2014-06-apug/2014-06-apug.ipynb @@ -0,0 +1,1198 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:0d2bad73107e13fa2f3e4ed1371a8c695080cd10f273423eba9655a52e0e63f1" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "DistArray: Distributed Arrays for Python\n", + "========================================\n", + "\n", + "Robert Grant, Enthought\n", + "\n", + "11 June 2014\n", + "\n", + "[github.com/enthought/distarray](https://github.com/enthought/distarray)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "source": [ + "Start a cluster\n", + "---------------\n", + "These examples require an `IPython.parallel` cluster to be running.\n", + "Outside the notebook, run\n", + "```\n", + "dacluster start -n4\n", + "```" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# some utility imports\n", + "from pprint import pprint\n", + "from matplotlib import pyplot as plt\n", + "\n", + "# main imports\n", + "import numpy\n", + "import distarray\n", + "\n", + "numpy.set_printoptions(precision=2) # display formatting" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "NumPy Arrays\n", + "------------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "nparr = numpy.random.random((4, 5))\n", + "nparr" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 2, + "text": [ + "array([[ 0.67, 0.27, 0.91, 0.05, 0.78],\n", + " [ 0.38, 0.07, 0.15, 0.7 , 0.64],\n", + " [ 0.82, 0.18, 0.58, 0.14, 0.97],\n", + " [ 0.76, 0.88, 0.5 , 0.34, 0.89]])" + ] + } + ], + "prompt_number": 2 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# NumPy array attributes\n", + "print \"type:\", type(nparr)\n", + "print \"dtype:\", nparr.dtype\n", + "print \"ndim:\", nparr.ndim\n", + "print \"shape:\", nparr.shape\n", + "print \"itemsize:\", nparr.itemsize\n", + "print \"nbytes:\", nparr.nbytes" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "type: \n", + "dtype: float64\n", + "ndim: 2\n", + "shape: (4, 5)\n", + "itemsize: 8\n", + "nbytes: 160\n" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "DistArrays\n", + "----------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from distarray.dist import Context\n", + "context = Context()\n", + "darr = context.fromarray(nparr)\n", + "darr" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 4, + "text": [ + "" + ] + } + ], + "prompt_number": 4 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# parts of the array are stored on each engine\n", + "for i, a in enumerate(darr.get_localarrays()):\n", + " print i, a" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "0 [[ 0.67 0.27 0.91 0.05 0.78]]\n", + "1 [[ 0.38 0.07 0.15 0.7 0.64]]\n", + "2 [[ 0.82 0.18 0.58 0.14 0.97]]\n", + "3 [[ 0.76 0.88 0.5 0.34 0.89]]\n" + ] + } + ], + "prompt_number": 5 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# DistArray attributes\n", + "print \"type:\", type(darr)\n", + "print \"dtype:\", darr.dtype\n", + "print \"ndim:\", darr.ndim\n", + "print \"shape:\", darr.shape\n", + "print \"itemsize:\", darr.itemsize\n", + "print \"nbytes:\", darr.nbytes" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "type: \n", + "dtype: float64\n", + "ndim: 2\n", + "shape: (4, 5)\n", + "itemsize: 8\n", + "nbytes: 160\n" + ] + } + ], + "prompt_number": 6 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# with some extra...\n", + "print \"targets:\", darr.targets\n", + "print \"context:\", darr.context \n", + "print \"distribution:\", darr.distribution" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "targets: [0, 1, 2, 3]\n", + "context: \n", + "distribution: \n" + ] + } + ], + "prompt_number": 7 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Universal Functions (ufuncs)\n", + "----------------------------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## NumPy ##\n", + "numpy.sin(nparr)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 8, + "text": [ + "array([[ 0.62, 0.27, 0.79, 0.05, 0.71],\n", + " [ 0.37, 0.07, 0.15, 0.65, 0.59],\n", + " [ 0.73, 0.18, 0.55, 0.14, 0.82],\n", + " [ 0.69, 0.77, 0.48, 0.34, 0.77]])" + ] + } + ], + "prompt_number": 8 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## DistArray ##\n", + "import distarray.dist as da\n", + "da.sin(darr)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 9, + "text": [ + "" + ] + } + ], + "prompt_number": 9 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "da.sin(darr).toarray()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 10, + "text": [ + "array([[ 0.62, 0.27, 0.79, 0.05, 0.71],\n", + " [ 0.37, 0.07, 0.15, 0.65, 0.59],\n", + " [ 0.73, 0.18, 0.55, 0.14, 0.82],\n", + " [ 0.69, 0.77, 0.48, 0.34, 0.77]])" + ] + } + ], + "prompt_number": 10 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## NumPy ##\n", + "nparr + nparr" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 11, + "text": [ + "array([[ 1.33, 0.55, 1.82, 0.1 , 1.57],\n", + " [ 0.77, 0.14, 0.3 , 1.41, 1.27],\n", + " [ 1.64, 0.36, 1.16, 0.28, 1.94],\n", + " [ 1.52, 1.76, 1.01, 0.69, 1.77]])" + ] + } + ], + "prompt_number": 11 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## DistArray ##\n", + "darr + darr" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 12, + "text": [ + "" + ] + } + ], + "prompt_number": 12 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "(darr + darr).toarray()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 13, + "text": [ + "array([[ 1.33, 0.55, 1.82, 0.1 , 1.57],\n", + " [ 0.77, 0.14, 0.3 , 1.41, 1.27],\n", + " [ 1.64, 0.36, 1.16, 0.28, 1.94],\n", + " [ 1.52, 1.76, 1.01, 0.69, 1.77]])" + ] + } + ], + "prompt_number": 13 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Distributions\n", + "-------------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# Distributions control which processes own which (global) indices\n", + "distribution = darr.distribution" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [], + "prompt_number": 14 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# this is a 2D distribution\n", + "pprint(distribution.maps)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "[,\n", + " ]\n" + ] + } + ], + "prompt_number": 15 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# setup\n", + "from distarray.plotting import plot_array_distribution\n", + "process_coords = [(0, 0), (1, 0), (2, 0), (3, 0)]" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [], + "prompt_number": 16 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "plot_array_distribution(darr, process_coords, cell_label=False, legend=True)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 17, + "text": [ + "" + ] + }, + { + "metadata": {}, + "output_type": "display_data", + "png": "iVBORw0KGgoAAAANSUhEUgAAAbUAAAEzCAYAAAC2Q50YAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAD69JREFUeJzt3X2MZXV9x/H3Z3eh0hLTWkwU2Dq2FV1AolYJirG7TWw2\nptGmqdKNtMY/+iiBmqbSB5vCX/7TB/6gJiYuSDUFW4ymSFbS4I7F0Cxi2JZ9wGLiNDxEbCrSIjHZ\nlW//mDs6PLj37s6dOYfvvF/Jzd57Z37nfDMh8+acc+feVBWSJHWwZegBJEmaF6MmSWrDqEmS2jBq\nkqQ2jJokqQ2jJklqo03UkuxO8kCSB5NcPfQ8Y5DkhiSPJbl/6FnGIsn2JPuTHE5yKMmVQ880Bkle\nlORAkoNJjiT5yNAzjUWSrUnuS3Lb0LNouhZRS7IVuB7YDZwP7EmyY9ipRuFGln8m+qFjwAer6gLg\nEuAD/rcCVfU9YFdVvQ64CNiV5K0DjzUWVwFHAP+o9wWgRdSAi4GvV9VSVR0DbgHeNfBMg6uqu4DH\nh55jTKrqm1V1cHL/SeAocPawU41DVT01uXs6sBX49oDjjEKSc4F3AB8HMvA4mkGXqJ0DPLTq8cOT\n56QfKckC8HrgwLCTjEOSLUkOAo8B+6vqyNAzjcDfAn8MPD30IJpNl6h5WkAnJcmZwK3AVZMjtk2v\nqp6enH48F3hbkp0DjzSoJL8CfKuq7sOjtBeMLlF7BNi+6vF2lo/WpOdIchrwGeBTVfW5oecZm6p6\nArgdeOPQswzsLcA7k3wDuBn4pSR/P/BMmqJL1O4FXpVkIcnpwGXAPw88k0YoSYC9wJGqum7oecYi\nyVlJfnJy/wzg7cB9w041rKr6s6raXlWvBH4D+GJV/dbQc+nEWkStqo4DVwB3sPwqpU9X1dFhpxpe\nkpuBu4HzkjyU5P1DzzQClwKXs/zqvvsmN18hCi8Hvji5pnYAuK2q7hx4prHxMscLQPzoGUlSFy2O\n1CRJAqMmSWrEqEmS2jBqkqQ2jJokqY1ta91AEl8+KUkbqKo27B1O1vI7fiPnXLHmqC37y/lsZs0W\ngZ0DzzBGi/hzeT6L+HN5tkX8mTyfRcbzc7l2w/d4zQatmQdPP0qS2jBqkqQ2mkVtYegBRmph6AFG\namHoAUZoYegBRmph6AE0I6O2KSwMPcBILQw9wAgtDD3ASC0MPYBm1CxqkqTNzKhJktowapKkNoya\nJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoya\nJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqY2pUUuyO8kDSR5McvVGDCVJ\n6iXJi5IcSHIwyZEkH1mP/Zwwakm2AtcDu4HzgT1JdqzHIJKkvqrqe8CuqnodcBGwK8lb572faUdq\nFwNfr6qlqjoG3AK8a95DSJL6q6qnJndPB7YC3573PqZF7RzgoVWPH548J0nSSUmyJclB4DFgf1Ud\nmfc+pkWt5r1DSdLmVFVPT04/ngu8LcnOee9j25SvPwJsX/V4O8tHa8+yuOr+wuQmSVq7pclt3L7B\n7FNW1RNJbgfeyDMDsmbTonYv8KokC8CjwGXAnud+2855ziRJ+oEFnnmg8KVhxpjilZPbimdPmeQs\n4HhVfSfJGcDbgWvnPccJo1ZVx5NcAdzB8kW9vVV1dN5DSJLaezlwU5ItLF/6+mRV3TnvnUw7UqOq\n9gH75r1jSdLmUVX3A29Y7/34jiKSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktow\napKkNoyaJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktow\napKkNoyaJKkNoyZJasOoSZLaMGqSpDaMmiSpDaMmSWrDqEmS2jBqkqQ2jJokqY1tc9nKoWvmshlJ\n0hQXXjv0BKOWqlrbBpLi0Nq2IUma0YWhqrJRu0tS9egprDubDZ1zhacfJUltGDVJUhtGTZLUhlGT\nJLVh1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGT\nJLVh1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGT\nJLVh1CRJbUyNWpIbkjyW5P6NGEiS1E+S7Un2Jzmc5FCSK9djP7Mcqd0I7F6PnUuSNo1jwAer6gLg\nEuADSXbMeydTo1ZVdwGPz3vHkqTNo6q+WVUHJ/efBI4CZ897P15TkyRtqCQLwOuBA/Pe9ra5bOXv\nrvnh/TfthIt3zmWzkrTp3bMIX1kceoqpFu9evk2T5EzgVuCqyRHbXKWqZhliAbitql77PF8rDk3f\nhiRpDi4MVZWN2l2SqkdPYd3ZPGfOJKcBnwf2VdV185nwmTz9KElad0kC7AWOrFfQYLaX9N8M3A2c\nl+ShJO9fr2EkSW1dClwO7Epy3+Q291fWT72mVlV75r1TSdLmUlVfZgPODnr6UZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNkjQaSbYl+dqprjdqkqTRqKrjwANJXnEq6+fz0TOS\nJM3PS4DDSe4Bvjt5rqrqndMWGjVJ0tj8xeTflc81y6r7J2TUJEmjUlWLSV4GvInlmN1TVd+aZa3X\n1CRJo5LkPcAB4N3Ae4B7krx7lrUeqUmSxubDwJtWjs6SvBS4E/inaQs9UpMkjU2A/171+H8mz03l\nkZokaWy+ANyR5B9YjtllwL5ZFho1SdLYfAj4NeDSyeOPVdVnZ1lo1CRJY/PjwOeq6jNJXg28Jslp\nVXVs2kKvqUmSxuYu4MeSnAPcAVwOfGKWhUZNkjQ2qaqnWD4F+dGqejdw4SwLjZokaXSSvBl4L3D7\n5KmZemXUJElj84fAnwKfrarDSX4O2D/LwlTN9HZaP3oDSXFobduQJM3owlBVM/3N1jwkqXr0FNad\nzVzmTLIFOLOq/neW7/dITZI0KkluTvLiJD8BHAKOJvnQLGuNmiRpbM6fHJn9Kst/dL0A/OYsC42a\nJGlstiU5jeWo3Tb5+7SZrnMZNUnS2HwMWALOBP41yQLwxCwLfaGIJL2QbLIXikxmCLC1qo5P+16P\n1CRJo5LkZUn2JvnC5KkdwPtmWTuX936sn9qw/2mQpE1tk/y2/QRwI/Dnk8cPAv8I7J220Dc0liSd\nUB4/lUtMa8rvWVX16SR/AlBVx5JMPfUInn6UJI3Pk0l+euVBkkuY8YUiHqlJksbmj4DbgJ9Ncjfw\nUuDXZ1lo1CRJo1JVX03yi8CrWT6P+cAsn6UGnn6UJI1MkitYfr/HQ1V1P3Bmkj+YZa1RkySNzW9X\n1eMrDyb3f2eWhUZNkjQ2Wybvzg9Akq3AabMs9JqaJGls7gBuSfIxlq+p/S7whRMvWWbUJEljczXL\npxt/f/L4X4CPz7LQqEmSRqWqvp/kBuDLk6ceqKrvz7LWqEmSRiXJTuAm4L8mT/1MkvdV1ZemrTVq\nkqSx+Rvgl6vqawBJzgNuAd4wbaGvfpQkjc22laABVNV/MuNBmEdqkqSx+WqSjwOfYvnVj+8F7p1l\noVGTJI3N7wFXAFdOHt8FfHSWhUZNkjQaSbYB/15VrwH++mTXe01NkjQaVXUc+FqSV5zKeo/UJElj\n8xLgcJJ7gO9Onquqeue0hUZNkjQ2H578u/rjs2f6+G2jJkkahSRnsPwikZ8H/gO4YdbPUVvhNTVJ\n0ljcBPwCy0F7B/BXJ7sBj9QkSWOxo6peC5BkL/CVk92AR2qSpLE4vnJn8irIk+aRmiRpLC5K8n+r\nHp+x6nFV1YunbcCoSZJGoaq2rnUbnn6UJLVh1CRJbRg1SVIbRk2S1IZRkyS1MTVqSbYn2Z/kcJJD\nSa6ctkaSpNWS3JDksST3r+d+ZjlSOwZ8sKouAC4BPpBkx3oOJUlq50Zg93rvZGrUquqbVXVwcv9J\n4Chw9noPJknqo6ruAh5f7/2c1DW1JAvA64ED6zGMJElrMfM7iiQ5E7gVuGpyxPYD16z6wO2db4ad\nb5nXeJK0uS3eDYv/NvQUM7hnEb6yOPQUpGr6564lOQ34PLCvqq571teqHlmn6SRJz5BzoKoy/Tvn\ntL+kODzT53M+0wV5zpyTs323rbwT/3qY5dWPAfYCR54dNEmSxmSWa2qXApcDu5LcN7mt+ytYJEl9\nJLkZuBs4L8lDSd6/HvuZek2tqr6Mf6QtSVqDqtqzEfsxVpKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpDaMmSWrDqEmS2jBqkqQ2jJokqQ2jJklqw6hJktowapKkNoyaJKkNoyZJasOoSZLaMGqSpDaM\nmiSpjW3z2Mg158xjK5Ikrc1coiZJauyCa4eeYGaefpQktWHUJEltGDVJUhtGTZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh1CRJ\nbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUxgmjluRFSQ4k\nOZjkSJKPbNRgkqRekuxO8kCSB5NcvR77OGHUqup7wK6qeh1wEbAryVvXYxBJUl9JtgLXA7uB84E9\nSXbMez9TTz9W1VOTu6cDW4Fvz3sISVJ7FwNfr6qlqjoG3AK8a947mRq1JFuSHAQeA/ZX1ZF5DyFJ\nau8c4KFVjx+ePDdXsxypPT05/Xgu8LYkO+c9hCSpvdqInWyb9Rur6okktwNvBBZXf231g4XJTZK0\ndkuT2/h9gymTPgJsX/V4O8tHa3N1wqglOQs4XlXfSXIG8Hbg2md/3855TyVJAp57oPClYcaYwSsn\ntxXPmfRe4FVJFoBHgcuAPfOeYtqR2suBm5JsYflU5Ser6s55DyFJ6q2qjie5AriD5Rcd7q2qo/Pe\nzwmjVlX3A2+Y904lSZtPVe0D9q3nPnxHEUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh\n1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktWHUJEltGDVJUhtGTZLUhlGTJLVh\n1CRJbRg1SVIbRk2S1IZRkyS1YdQkSW0YNUlSG0ZNktSGUZMktdEqaktDDzBSS0MPMFJLQw8wQktD\nDzBSS0MPoJkZtU1gaegBRmpp6AFGaGnoAUZqaegBNLNWUZMkbW5GTZLURqpqbRtI1rYBSdJJqaps\n1L7W8jt+I+dcseaoSZI0Fp5+lCS1YdQkSW0YNUlSG0ZNktSGUZMktfH/OydIpv9Xa0UAAAAASUVO\nRK5CYII=\n", + "text": [ + "" + ] + } + ], + "prompt_number": 17 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "distribution.maps[0].bounds" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 18, + "text": [ + "[(0, 1), (1, 2), (2, 3), (3, 4)]" + ] + } + ], + "prompt_number": 18 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# the above is the default, you can make more complex distributions\n", + "from distarray.dist import Distribution\n", + "distribution = Distribution.from_shape(context, (64, 64), dist=('b', 'c'))\n", + "a = context.zeros(distribution, dtype='int32')\n", + "plot_array_distribution(a, process_coords, cell_label=False, legend=True)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 19, + "text": [ + "" + ] + }, + { + "metadata": {}, + "output_type": "display_data", + "png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEzCAYAAAAlyyi6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFLdJREFUeJzt3X+MZXd93vH3410TDDRYC2jXGJdx2ywxENc2ZSGGhDGy\nkRNFG6sqJlaKVgjRtDWFpoiypGkz/qMCquKEKjZCie2unNSEmnplixh7cTwuyJXXhl38c72h9Ua2\nYcdWcRx+CNWGT/+4Z9jJenbnzsy9c8989/2Sruacc8/3nmet2cdnv/fcc1NVSJLacdKkA0iSRsti\nl6TGWOyS1BiLXZIaY7FLUmMsdklqzFiLPcnFSQ4k+cskHxvnsVYiybVJ5pI8sGDbpiR7khxMcnuS\nUyeZcaEkZyS5M8lDSR5M8qFue+8yJ3lxknuS7E/ycJJP9DXrQkk2JNmX5JZuvZd5kxxKcn+XdW+3\nrZdZAZKcmuTGJI90vw9v6XPe9W5sxZ5kA/CHwMXA64HLkpw1ruOt0HUM8i20E9hTVVuBO7r1vngO\n+O2qegPwVuDy7r9p7zJX1Y+AC6rqHOBs4IIkb6eHWY/yYeBhYP4DHn3NW8B0VZ1bVdu6bX3NCvAZ\n4M+r6iwGvw8H6Hfe9a2qxvIAfhH48oL1ncDOcR1vFTmngAcWrB8ANnfLW4ADk854nOy7gQv7nhl4\nCXAv8IY+ZwVeA3wFuAC4pc+/D8BjwCuO2tbXrC8H/s8i23uZt4XHOKdiTgceX7D+RLet7zZX1Vy3\nPAdsnmSYY0kyBZwL3ENPMyc5Kcl+BpnurKqH6GnWzu8DHwV+smBbX/MW8JUk9yX5QLetr1nPBJ5O\ncl2SbyT5oyQvpb95171xFvu6v1dBDU4levfnSPIy4IvAh6vqewuf61PmqvpJDaZiXgP8cpILjnq+\nN1mT/BrwVFXtA7LYPn3KC7ytqs4FfoXBlNwvLXyyZ1k3AucBV1fVecAPOGrapWd5171xFvuTwBkL\n1s9gcNbed3NJtgAkOQ14asJ5/pYkJzMo9eurane3udeZq+pZ4EvAm+hv1vOB7UkeA24A3pnkenqa\nt6q+0/18GrgJ2EZPszL4e/9EVd3brd/IoOgP9zTvujfOYr8P+LkkU0leBLwHuHmMxxuVm4Ed3fIO\nBvPYvZAkwDXAw1X1Bwue6l3mJK+cv8ohySnARcA+epgVoKp+p6rOqKozgd8A/qKq3ksP8yZ5SZK/\n0y2/FHgX8AA9zApQVYeBx5Ns7TZdCDwE3EIP8zZhnBP4DP6Z+CjwLeDjk35DYZF8NwDfBv4fg/cD\n3gdsYvAG2kHgduDUSedckPftDOZ/9zMoyX0MrurpXWbgF4BvdFnvBz7abe9d1kWyvwO4ua95GcxZ\n7+8eD87/3epj1gWZ/yGDN9C/CfwPBm+o9jbven+k+48uSWqEnzyVpMZY7JLUmFUVe99vGSBJJ6IV\nz7F3twx4lME73E8yeGPksqp6ZHTxJEnLtZoz9m3At6rqUFU9B3we+PXRxJIkrdTGVYxd7JYBb1m4\nQxIvuZE0FlW16CeEx2E1XbaWOeetptiH/IP+HjALTMODMy98+o1HbTt6n6OfH9U+x80yy6J5J5Jl\nqdeYBaZ7kmWYfWYZOu/Ef19mOWbWNc9ynOd/us8sI/1dWFWWYV5jlhXnfeOadyUzS+4xmjGjsJpi\nH/KWAbPAocHPvbOwbXoVh5R0YjoEV81MOsS6sZpi/+ktAxh8evM9wGUv3G2an/6f2VKXtCJTcPnM\nkdXPXjGpIOvCiou9qp5P8kHgNmADcM2xr4iZWulhJmRq0gGWYWrSAZZpatIBlmFq0gGWaWrSAZZp\natIBmrWaM3aq6lbg1qX3nFrNYSZgatIBlmFq0gGWaWrSAZZhatIBlmlq0gGWaWrSAZrlJ08lqTEW\nuyQ1xmKXpMZY7JLUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FL\nUmMsdklqjMUuSY2x2CWpMRa7JDXGYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglaY0keXGS\ne5LsT/Jwkk+M4zgbx/GikqQXqqofJbmgqn6YZCPwtSRvr6qvjfI4nrFL0hqqqh92iy8CNgDfHfUx\nLHZJWkNJTkqyH5gD7qyqh0d9DItdktZQVf2kqs4BXgP8cpLpUR/DOXZJGpHHgEND7ltVzyb5EvCP\ngNlR5rDYJWlEzuwe8+466vkkrwSer6q/TnIKcBFwxahzWOyStHZOA3YlOYnBVPj1VXXHqA9isUvS\nGqmqB4Dzxn0c3zyVpMZY7JLUmCWLPcm1SeaSPLBg26Yke5IcTHJ7klPHG1OSNKxhztivAy4+attO\nYE9VbQXu6NYlST2wZLFX1VeBZ47avB3Y1S3vAi4ZcS5J0gqtdI59c1XNdctzwOYR5ZEkrdKq3zyt\nqgJqBFkkSSOw0uvY55JsqarDSU4Dnjr2rrNHFvfOwrbpFR5S0onrEFw1M+kQ68ZKi/1mYAfwqe7n\n7mPvOn1k0VKXtCJTcPnMkdXPjvxT+E0Z5nLHG4C7gdcleTzJ+4BPAhclOQi8s1uXJPXAkmfsVXXZ\nMZ66cMRZJEkj4CdPJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXG\nYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglqTEWuyQ1xmKXpMZY7JLUmCW/zFqSBL/37eWP\nmXn16HMMwzN2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWp\nMRa7JDXGYpekxljsktSYJYs9yRlJ7kzyUJIHk3yo274pyZ4kB5PcnuTU8ceVJC1lmDP254Dfrqo3\nAG8FLk9yFrAT2FNVW4E7unVJ0oQtWexVdbiq9nfL3wceAU4HtgO7ut12AZeMK6QkaXjLmmNPMgWc\nC9wDbK6que6pOWDzSJNJklZk6K/GS/Iy4IvAh6vqe0l++lxVVZJafOTskcW9s7BteiU5JZ3QDsFV\nM5MOsW4MVexJTmZQ6tdX1e5u81ySLVV1OMlpwFOLj54+smipS1qRKbh85sjqZ6+YVJB1YZirYgJc\nAzxcVX+w4KmbgR3d8g5g99FjJUlrb5gz9rcB/xS4P8m+btvHgU8CX0jyfuAQcOlYEkqSlmXJYq+q\nr3HsM/sLRxtHkrRafvJUkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6Q1cqzboI/a0PeKkSSt\n2vxt0Pd399/6epI9VfXIKA/iGbskrZFj3Ab91aM+jsUuSRNw1G3QR8qpGEkakdm7B4+ldNMwNzK4\nDfr3R53DYpekEZk+f/CYd8WVL9xnwW3Q/2TBbdBHyqkYSVojx7kN+khZ7JK0duZvg35Bkn3d4+JR\nH8SpGElaI0vcBn1kPGOXpMZY7JLUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakxFrskNcZil6SeSbIx\nyaMrHW+xS1LPVNXzwIEkr13JeG8pIEn9tAl4KMle4Afdtqqq7UsNtNglqZ/+ffezup9ZsHxcFrsk\n9VBVzSbZAryZQaHvraqnhhnrHLsk9VCSSxl8bd67gUuBvUnePcxYz9glqZ9+F3jz/Fl6klcBdwD/\nfamBnrFLUj8FeHrB+v/tti3JM3ZJ6qcvA7cl+W8MCv09wK3DDLTYJamf/i3wjxl8nR7A56rqpmEG\nWuyS1E8vAXZX1ReTvA74+SQnV9VzSw087hx7khcnuSfJ/iQPJ/lEt31Tkj1JDia5Pcmpo/lzSJI6\nXwV+JsnpwG0MvgT7vw4z8LjFXlU/Ai6oqnOAsxl8s/bbgZ3AnqrayuBd2p0rzy5JWkSq6ocMpmOu\nrqp3A28cZuCSV8V0LwzwImAD8AywHdjVbd8FXLLcxJKk40vyi8BvAl/qNg11JeOSOyU5Kcl+YA64\ns6oeAjZX1Vy3yxywefmRJUnH8a+BjwM3VdVDSf4+cOcwA5d887SqfgKck+TlDC69ueCo5yvJUPcv\nkCQNp6ruAu6CwQk28HRVfWiYsUNfFVNVzyb5EvAmYC7Jlqo6nOQ04Dj3L5g9srh3FrZND3tISeoc\ngqtmJh1iTSW5Afgt4MfAvcDLk3ymqv7TUmOXuirmlfNXvCQ5BbgI2AfcDOzodtsB7D72q0wfeVjq\nklZkCi6fOfI4Mby+qv6GwXuYtwJTwHuHGbjUGftpwK7unwEnAddX1R1J9gFfSPJ+4BCDG9RIkkZn\nY5KTGRT7VVX13LDT3sct9qp6ADhvke3fBS5cSVJJ0lA+x+DE+X7gfyaZAp4dZqA3AZOkHqqq/1JV\np1fVr3QXsfwVcMFS48Bil6ReSrIlyTVJvtxtOosj720ef2zV+K5UTFL15JH1mdNfuM/Mk0etn378\n50e1j1nMYpb1m+UKoKqGuoXtKCSp+vYKxr165Tm7Qr8O+HdVdXY3376vqpb89Kk3AZOkIeSZlZwE\nr+r/Pa+sqj9LshOge/P0+WEGOhUjSf30/SSvmF9J8laGfPPUM3ZJ6qePALcAfy/J3cCrgH8yzECL\nXZJ6qKq+nuQdwOsYzOkcGOZe7OBUjCT1UpIPAi+rqge7zxS9LMm/HGasxS5J/fSBqnpmfqVb/mfD\nDLTYJamfTupu5wJAkg3AycMMdI5dkvrpNuDzST7HYI79t4AvH3/IgMUuSf30MQZTL/+iW98D/PEw\nAy12SeqhqvpxkmuBr3WbDlTVj4cZa7FLUg8lmWbwndJ/1W36u0l2dN+sdFwWuyT105XAu6rqUYAk\nW4HPs8it1I/mVTGS1E8b50sdoKoOMuTJuGfsktRPX0/yx8CfMLgq5jeB+4YZaLFLUj/9c+CDwIe6\n9a8CVw8z0GKXpJ5JshH4ZlX9PPDp5Y53jl2SeqaqngceTfLalYz3jF2S+mkT8FCSvcAPum1VVduX\nGmixS1I//W73c+HXMA31NU4WuyT1SJJTGLxx+g+A+4Frh70P+zzn2CWpX3YBb2JQ6r8K/OflvoBn\n7JLUL2dV1S8AJLkGuHe5L+AZuyT1y/PzC93VMcvmGbsk9cvZSb63YP2UBetVVT+71AtY7JLUI1W1\nYbWv4VSMJDXGYpekxljsktQYi12SGmOxS1JjLHZJWiNJrk0yl+SBcR7HYpektXMdcPG4DzJUsSfZ\nkGRfklu69U1J9iQ5mOT2JKeON6YkrX9V9VXgmXEfZ9gz9g8DD3PklpE7gT1VtRW4o1uXJPXAkp88\nTfIaBncY+4/Av+k2bwfe0S3vAmax3CWd6PbOwr2zk04x1C0Ffh/4KLDw/gSbq2quW54DNo86mCSt\nO9umB495V18xkRjHLfYkvwY8VVX7kkwvtk9VVZJjfqvHzIKvYT0ETK0gpKQT2+zdg2kBDWepM/bz\nge1JfhV4MfCzSa4H5pJsqarDSU4DnjrWC8x8ZMHylasPLOnEM30+TC9Yv2tSQVYpyQ0MprFfkeRx\n4D9U1XWjPs5x3zytqt+pqjOq6kzgN4C/qKr3AjcDO7rddgC7Rx1MklpTVZdV1aur6me6bh15qcPy\nr2Ofn3L5JHBRkoPAO7t1SVIPDH0/9qq6i+5fQFX1XeDCcYWSJK2cnzyVpMZY7JLUGItdkhpjsUtS\nYyx2SWqMxS5JjbHYJakxFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXG\nYpekxljsktQYi12SGmOxS1JjLHZJaozFLkmNsdglqTEWuyQ1xmKXpMZY7JLUGItdkhpjsUtSYyx2\nSWrMxkkHkKR14Q1XTDrB0Dxjl6TGWOyS1BiLXZIaY7FLUmOGevM0ySHgb4AfA89V1bYkm4A/A14L\nHAIuraq/HlNOSdKQhj1jL2C6qs6tqm3dtp3AnqraCtzRrUuSJmw5UzE5an07sKtb3gVcMpJEkqRV\nWc4Z+1eS3JfkA922zVU11y3PAZtHnk6StGzDfkDpbVX1nSSvAvYkObDwyaqqJDX6eJKk5Rqq2Kvq\nO93Pp5PcBGwD5pJsqarDSU4Dnlps7MynjywfAqZWl1fSCWj2bpiddIh1ZMliT/ISYENVfS/JS4F3\nAVcANwM7gE91P3cvNn7mIwuWr1x9YEknnunzYXrB+l2TCrJODHPGvhm4Kcn8/n9aVbcnuQ/4QpL3\n013uOLaUkqShLVnsVfUYcM4i278LXDiOUJKklfOTp5LUGItdkhpjsUtSYyx2SWqMxS5JjbHYJakx\nFrskNcZil6TGWOyS1BiLXZIaY7FLUmMsdklqjMUuSY2x2CWpMRa7JDXGYpekNZTk4iQHkvxlko+N\n4xgWuyStkSQbgD8ELgZeD1yW5KxRH8dil6S1sw34VlUdqqrngM8Dvz7qg1jskrR2TgceX7D+RLdt\npCx2SVo7tRYHWfLLrCVJw3oMOHS8HZ4EzliwfgaDs/aRstglaWTO7B7z7jp6h/uAn0syBXwbeA9w\n2ahTWOyStEaq6vkkHwRuAzYA11TVI6M+jsUuSWuoqm4Fbh3nMXzzVJIaY7FLUmMsdklqjMUuSY2x\n2CWpMRa7JDXGYpekxljsktQYi12SGmOxS1JjLHZJasxQxZ7k1CQ3JnkkycNJ3pJkU5I9SQ4muT3J\nqeMOK0la2rBn7J8B/ryqzgLOBg4AO4E9VbUVuKNblyRN2JLFnuTlwC9V1bUwuO1kVT0LbAd2dbvt\nAi4ZW0pJ0tCGOWM/E3g6yXVJvpHkj5K8FNhcVXPdPnPA5rGllCQNbZhi3wicB1xdVecBP+CoaZeq\nKtbou/wkScc3zBdtPAE8UVX3dus3Ah8HDifZUlWHk5wGPLXY4JlPH1k+BEytIqykE9Ps3TA76RDr\nyJLF3hX340m2VtVB4ELgoe6xA/hU93P3YuNnPrJg+coRJJZ0wpk+H6YXrL/gm0T1twz71Xj/CvjT\nJC8C/jfwPgbf1/eFJO9ncDJ+6VgSSpKWZahir6pvAm9e5KkLRxtHkrRafvJUkhpjsUtSYyx2SWqM\nxS5JjbHYJakxFrskNcZil6TGrEmxz969FkcZnfWUdz1lhfWVdz1lBfPqiLUp9v+1FkcZnfWUdz1l\nhfWVdz1lBfPqCKdiJKkxFrskNSaDW6mP6cUT79EuaSyqKmt1rNV02VrmnDfWYpckrT2nYiSpMRa7\nJDXGYpekxljsktQYi12SGvP/AdzJez1KiumSAAAAAElFTkSuQmCC\n", + "text": [ + "" + ] + } + ], + "prompt_number": 19 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Contexts\n", + "--------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# Context objects manage the setup and communication of the worker processes\n", + "# for DistArray objects. \n", + "print \"targets:\", context.targets\n", + "print \"comm:\", context.comm" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "targets: [0, 1, 2, 3]\n", + "comm: __distarray__e60fe9953c015b300\n" + ] + } + ], + "prompt_number": 20 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "IO Support (v0.2)\n", + "-----------------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# load .npy files in parallel\n", + "numpy.save(\"/tmp/outfile.npy\", nparr)\n", + "distribution = Distribution.from_shape(context, nparr.shape) \n", + "new_darr = context.load_npy(\"/tmp/outfile.npy\", distribution)\n", + "new_darr" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 21, + "text": [ + "" + ] + } + ], + "prompt_number": 21 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# save DistArrays to .hdf5 files in parallel\n", + "context.save_hdf5(\"/tmp/outfile.hdf5\", darr, mode='w')" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [], + "prompt_number": 22 + }, + { + "cell_type": "code", + "collapsed": true, + "input": [ + "# load DistArrays from .hdf5 files in parallel (using h5py)\n", + "context.load_hdf5(\"/tmp/outfile.hdf5\", distribution)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 23, + "text": [ + "" + ] + } + ], + "prompt_number": 23 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# save to .dnpy (a built-in flat-file format based on .npy)\n", + "context.save_dnpy(\"/tmp/outfile\", darr)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [], + "prompt_number": 24 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# load from .dnpy\n", + "context.load_dnpy(\"/tmp/outfile\")" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 25, + "text": [ + "" + ] + } + ], + "prompt_number": 25 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Reductions (v0.3)\n", + "-----------------" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## NumPy ##\n", + "print \"sum:\", nparr.sum()\n", + "print \"sum over an axis:\", nparr.sum(axis=1)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "sum: 10.6867002882\n", + "sum over an axis: [ 2.68 1.94 2.69 3.37]\n" + ] + } + ], + "prompt_number": 26 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "## DistArray ##\n", + "print \"sum:\", darr.sum(), darr.sum().toarray()\n", + "print \"sum over an axis:\", darr.sum(axis=1), darr.sum(axis=1).toarray()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "sum: " + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "10.6867002882\n", + "sum over an axis: " + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "[ 2.68 1.94 2.69 3.37]\n" + ] + } + ], + "prompt_number": 27 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Context.apply (v0.3)\n", + "--------------------\n", + "Global view, local control" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def get_local_random():\n", + " import numpy\n", + " return numpy.random.randint(10)\n", + "\n", + "context.apply(get_local_random)" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 28, + "text": [ + "[2, 4, 6, 5]" + ] + } + ], + "prompt_number": 28 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def get_local_var(darr):\n", + " return darr.ndarray.var()\n", + "\n", + "context.apply(get_local_var, args=(darr.key,))" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 29, + "text": [ + "[0.1043967698165971,\n", + " 0.064006352164566974,\n", + " 0.11124749755337127,\n", + " 0.046561405334938577]" + ] + } + ], + "prompt_number": 29 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Distributed Slicing (v0.4)\n", + "--------------------------\n", + "(coming soon)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# as a reminder\n", + "darr.toarray()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 30, + "text": [ + "array([[ 0.67, 0.27, 0.91, 0.05, 0.78],\n", + " [ 0.38, 0.07, 0.15, 0.7 , 0.64],\n", + " [ 0.82, 0.18, 0.58, 0.14, 0.97],\n", + " [ 0.76, 0.88, 0.5 , 0.34, 0.89]])" + ] + } + ], + "prompt_number": 30 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "darr.get_localshapes()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 31, + "text": [ + "[(1, 5), (1, 5), (1, 5), (1, 5)]" + ] + } + ], + "prompt_number": 31 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# take a column slice\n", + "darr_view = darr[:, 3]\n", + "print darr_view\n", + "print darr_view.toarray()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "[ 0.05 0.7 0.14 0.34]\n" + ] + } + ], + "prompt_number": 32 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# changes in the view change the original\n", + "darr_view[3] = -0.99\n", + "print \"view:\"\n", + "print darr_view.toarray()\n", + "\n", + "print \"original:\"\n", + "print darr.toarray()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "view:\n", + "[ 0.05 0.7 0.14 -0.99]\n", + "original:\n", + "[[ 0.67 0.27 0.91 0.05 0.78]\n", + " [ 0.38 0.07 0.15 0.7 0.64]\n", + " [ 0.82 0.18 0.58 0.14 0.97]\n", + " [ 0.76 0.88 0.5 -0.99 0.89]]\n" + ] + } + ], + "prompt_number": 33 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# a more complex slice\n", + "print darr[:, 2::2]\n", + "print darr[:-1, 2::2].toarray()" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "[[ 0.91 0.78]\n", + " [ 0.15 0.64]\n", + " [ 0.58 0.97]]" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n" + ] + } + ], + "prompt_number": 34 + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Distributed Array Protocol (v0.2)\n", + "---------------------------------\n", + "(exporting and importing distributed arrays)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def return_protocol_structure(darr):\n", + " return darr.__distarray__()\n", + "\n", + "context.apply(return_protocol_structure, args=(darr.key,))" + ], + "language": "python", + "metadata": { + "slideshow": { + "slide_type": "subslide" + } + }, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 35, + "text": [ + "[{'__version__': '0.10.0',\n", + " 'buffer': array([[ 0.67, 0.27, 0.91, 0.05, 0.78]]),\n", + " 'dim_data': ({'dist_type': 'b',\n", + " 'proc_grid_rank': 0,\n", + " 'proc_grid_size': 4,\n", + " 'size': 4,\n", + " 'start': 0,\n", + " 'stop': 1},\n", + " {'dist_type': 'b',\n", + " 'proc_grid_rank': 0,\n", + " 'proc_grid_size': 1,\n", + " 'size': 5,\n", + " 'start': 0,\n", + " 'stop': 5})},\n", + " {'__version__': '0.10.0',\n", + " 'buffer': array([[ 0.38, 0.07, 0.15, 0.7 , 0.64]]),\n", + " 'dim_data': ({'dist_type': 'b',\n", + " 'proc_grid_rank': 1,\n", + " 'proc_grid_size': 4,\n", + " 'size': 4,\n", + " 'start': 1,\n", + " 'stop': 2},\n", + " {'dist_type': 'b',\n", + " 'proc_grid_rank': 0,\n", + " 'proc_grid_size': 1,\n", + " 'size': 5,\n", + " 'start': 0,\n", + " 'stop': 5})},\n", + " {'__version__': '0.10.0',\n", + " 'buffer': array([[ 0.82, 0.18, 0.58, 0.14, 0.97]]),\n", + " 'dim_data': ({'dist_type': 'b',\n", + " 'proc_grid_rank': 2,\n", + " 'proc_grid_size': 4,\n", + " 'size': 4,\n", + " 'start': 2,\n", + " 'stop': 3},\n", + " {'dist_type': 'b',\n", + " 'proc_grid_rank': 0,\n", + " 'proc_grid_size': 1,\n", + " 'size': 5,\n", + " 'start': 0,\n", + " 'stop': 5})},\n", + " {'__version__': '0.10.0',\n", + " 'buffer': array([[ 0.76, 0.88, 0.5 , -0.99, 0.89]]),\n", + " 'dim_data': ({'dist_type': 'b',\n", + " 'proc_grid_rank': 3,\n", + " 'proc_grid_size': 4,\n", + " 'size': 4,\n", + " 'start': 3,\n", + " 'stop': 4},\n", + " {'dist_type': 'b',\n", + " 'proc_grid_rank': 0,\n", + " 'proc_grid_size': 1,\n", + " 'size': 5,\n", + " 'start': 0,\n", + " 'stop': 5})}]" + ] + } + ], + "prompt_number": 35 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Acknowledgement and Disclaimer\n", + "------------------------------\n", + "\n", + "This material is based upon work supported by the Department of Energy under Award Number DE-SC0007699.\n", + "\n", + "This report was prepared as an account of work sponsored by an agency of the United States Government. Neither the United States Government nor any agency thereof, nor any of their employees, makes any warranty, express or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness of any information, apparatus, product, or process disclosed, or represents that its use would not infringe privately owned rights. Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or otherwise does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government or any agency thereof. The views and opinions of authors expressed herein do not necessarily state or reflect those of the United States Government or any agency thereof. " + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/docs/2014-06-apug/2014-06-apug.pdf b/docs/2014-06-apug/2014-06-apug.pdf new file mode 100644 index 00000000..be6a3b54 Binary files /dev/null and b/docs/2014-06-apug/2014-06-apug.pdf differ diff --git a/docs/2014-06-apug/custom.css b/docs/2014-06-apug/custom.css new file mode 100644 index 00000000..ecd6c375 --- /dev/null +++ b/docs/2014-06-apug/custom.css @@ -0,0 +1,5 @@ +/* make the default font-size bigger in slides mode */ + +.reveal { + font-size: 250%; +}