diff --git a/distarray/__init__.py b/distarray/__init__.py index 4dadf681..4c53159f 100644 --- a/distarray/__init__.py +++ b/distarray/__init__.py @@ -5,7 +5,4 @@ # --------------------------------------------------------------------------- __version__ = "0.3.0-dev" - -from distarray.client import DistArray -from distarray.context import Context -from distarray.functions import * +DISTARRAY_BASE_NAME = '__distarray__' diff --git a/distarray/apps/dacluster.py b/distarray/apps/dacluster.py index f5e08dc5..a44b4e54 100755 --- a/distarray/apps/dacluster.py +++ b/distarray/apps/dacluster.py @@ -18,7 +18,7 @@ from subprocess import Popen, PIPE from distarray.externals import six -from distarray.cleanup import clear_all +from distarray.dist.cleanup import clear_all if six.PY2: diff --git a/distarray/dist/__init__.py b/distarray/dist/__init__.py new file mode 100644 index 00000000..8590522a --- /dev/null +++ b/distarray/dist/__init__.py @@ -0,0 +1,11 @@ +# encoding: utf-8 +# --------------------------------------------------------------------------- +# Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc. +# Distributed under the terms of the BSD License. See COPYING.rst. +# --------------------------------------------------------------------------- + +from __future__ import absolute_import + +from distarray.dist.distarray import DistArray +from distarray.dist.context import Context +from distarray.dist.functions import * diff --git a/distarray/cleanup.py b/distarray/dist/cleanup.py similarity index 95% rename from distarray/cleanup.py rename to distarray/dist/cleanup.py index 06d444ab..8e967c69 100644 --- a/distarray/cleanup.py +++ b/distarray/dist/cleanup.py @@ -4,9 +4,9 @@ # Distributed under the terms of the BSD License. See COPYING.rst. # --------------------------------------------------------------------------- -from __future__ import print_function +from __future__ import print_function, absolute_import -from distarray.ipython_utils import IPythonClient +from distarray.dist.ipython_utils import IPythonClient def cleanup(view, module_name, prefix): """ Delete Context object with the given name from the given module""" @@ -46,7 +46,7 @@ def get_keys_engine(prefix): for target, keys in keys_from_target.items(): for key in keys: targets_from_key.setdefault(key, []).append(target) - + return targets_from_key def clear(view): diff --git a/distarray/context.py b/distarray/dist/context.py similarity index 97% rename from distarray/context.py rename to distarray/dist/context.py index 74434670..7f3cc7c9 100644 --- a/distarray/context.py +++ b/distarray/dist/context.py @@ -8,6 +8,7 @@ communicate with localarrays. """ +from __future__ import absolute_import import uuid import collections @@ -15,14 +16,13 @@ import numpy -from distarray import cleanup +from distarray.dist import cleanup from distarray.externals import six -from distarray.client import DistArray -from distarray.client_map import Distribution -from distarray.ipython_utils import IPythonClient +from distarray.dist.distarray import DistArray +from distarray.dist.maps import Distribution - -DISTARRAY_BASE_NAME = '__distarray__' +from distarray.dist.ipython_utils import IPythonClient +from distarray import DISTARRAY_BASE_NAME class Context(object): @@ -70,7 +70,7 @@ def __init__(self, client=None, targets=None): #with self.view.sync_imports(): # import distarray self.view.execute("import distarray.local; " - "import distarray.mpiutils; " + "import distarray.local.mpiutils; " "import numpy") self.context_key = self._setup_context_key() @@ -91,13 +91,13 @@ def _setup_context_key(self): def _make_intracomm(self): def get_rank(): - from distarray.mpiutils import COMM_PRIVATE + from distarray.local.mpiutils import COMM_PRIVATE return COMM_PRIVATE.Get_rank() # self.view's engines must encompass all ranks in the MPI communicator, # i.e., everything in rank_map.values(). def get_size(): - from distarray.mpiutils import COMM_PRIVATE + from distarray.local.mpiutils import COMM_PRIVATE return COMM_PRIVATE.Get_size() # get a mapping of IPython engine ID to MPI rank @@ -112,7 +112,7 @@ def get_size(): # MPI_Comm_create must be called on all engines, not just those # involved in the new communicator. comm_key = self._generate_key() - cmd = "%s = distarray.mpiutils.create_comm_with_list(%s)" + cmd = "%s = distarray.local.mpiutils.create_comm_with_list(%s)" cmd %= (comm_key, ranks) self.view.execute(cmd, block=True) return comm_key diff --git a/distarray/decorators.py b/distarray/dist/decorators.py similarity index 98% rename from distarray/decorators.py rename to distarray/dist/decorators.py index 3dd60150..37427807 100644 --- a/distarray/decorators.py +++ b/distarray/dist/decorators.py @@ -8,10 +8,12 @@ Decorators for defining functions that use `DistArrays`. """ +from __future__ import absolute_import + import functools -from distarray.client import DistArray -from distarray.context import Context +from distarray.dist.distarray import DistArray +from distarray.dist.context import Context from distarray.error import ContextError from distarray.utils import has_exactly_one diff --git a/distarray/client.py b/distarray/dist/distarray.py similarity index 80% rename from distarray/client.py rename to distarray/dist/distarray.py index 0276285a..aa87e7c3 100644 --- a/distarray/client.py +++ b/distarray/dist/distarray.py @@ -12,13 +12,15 @@ # Imports # --------------------------------------------------------------------------- +from __future__ import absolute_import + import operator from itertools import product import numpy as np import distarray -from distarray.client_map import Distribution +from distarray.dist.maps import Distribution from distarray.externals.six import next from distarray.utils import has_exactly_one, _raise_nie @@ -351,113 +353,113 @@ def _rbinary_op_from_ufunc(self, other, func, lop_str, *args, **kwargs): return func(other, self, *args, **kwargs) def __add__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.add, '__radd__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.add, '__radd__', *args, **kwargs) def __sub__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.subtract, '__rsub__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.subtract, '__rsub__', *args, **kwargs) def __mul__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.multiply, '__rmul__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.multiply, '__rmul__', *args, **kwargs) def __div__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.divide, '__rdiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.divide, '__rdiv__', *args, **kwargs) def __truediv__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.true_divide, '__rtruediv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.true_divide, '__rtruediv__', *args, **kwargs) def __floordiv__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.floor_divide, '__rfloordiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.floor_divide, '__rfloordiv__', *args, **kwargs) def __mod__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.mod, '__rdiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.mod, '__rdiv__', *args, **kwargs) def __pow__(self, other, modulo=None, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.power, '__rpower__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.power, '__rpower__', *args, **kwargs) def __lshift__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.left_shift, '__rlshift__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.left_shift, '__rlshift__', *args, **kwargs) def __rshift__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.right_shift, '__rrshift__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.right_shift, '__rrshift__', *args, **kwargs) def __and__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.bitwise_and, '__rand__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.bitwise_and, '__rand__', *args, **kwargs) def __or__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.bitwise_or, '__ror__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.bitwise_or, '__ror__', *args, **kwargs) def __xor__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.bitwise_xor, '__rxor__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.bitwise_xor, '__rxor__', *args, **kwargs) # Binary - right versions def __radd__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.add, '__add__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.add, '__add__', *args, **kwargs) def __rsub__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.subtract, '__sub__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.subtract, '__sub__', *args, **kwargs) def __rmul__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.multiply, '__mul__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.multiply, '__mul__', *args, **kwargs) def __rdiv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.divide, '__div__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.divide, '__div__', *args, **kwargs) def __rtruediv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.true_divide, '__truediv__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.true_divide, '__truediv__', *args, **kwargs) def __rfloordiv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.floor_divide, '__floordiv__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.floor_divide, '__floordiv__', *args, **kwargs) def __rmod__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.mod, '__mod__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.mod, '__mod__', *args, **kwargs) def __rpow__(self, other, modulo=None, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.power, '__pow__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.power, '__pow__', *args, **kwargs) def __rlshift__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.left_shift, '__lshift__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.left_shift, '__lshift__', *args, **kwargs) def __rrshift__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.right_shift, '__rshift__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.right_shift, '__rshift__', *args, **kwargs) def __rand__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.bitwise_and, '__and__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.bitwise_and, '__and__', *args, **kwargs) def __ror__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.bitwise_or, '__or__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.bitwise_or, '__or__', *args, **kwargs) def __rxor__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.bitwise_xor, '__xor__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.bitwise_xor, '__xor__', *args, **kwargs) def __neg__(self, *args, **kwargs): - return distarray.negative(self, *args, **kwargs) + return distarray.dist.negative(self, *args, **kwargs) def __pos__(self, *args, **kwargs): return self def __abs__(self, *args, **kwargs): - return distarray.abs(self, *args, **kwargs) + return distarray.dist.abs(self, *args, **kwargs) def __invert__(self, *args, **kwargs): - return distarray.invert(self, *args, **kwargs) + return distarray.dist.invert(self, *args, **kwargs) # Boolean comparisons def __lt__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.less, '__lt__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.less, '__lt__', *args, **kwargs) def __le__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.less_equal, '__le__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.less_equal, '__le__', *args, **kwargs) def __eq__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.equal, '__eq__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.equal, '__eq__', *args, **kwargs) def __ne__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.not_equal, '__ne__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.not_equal, '__ne__', *args, **kwargs) def __gt__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.greater, '__gt__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.greater, '__gt__', *args, **kwargs) def __ge__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.greater_equal, '__ge__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.greater_equal, '__ge__', *args, **kwargs) diff --git a/distarray/functions.py b/distarray/dist/functions.py similarity index 97% rename from distarray/functions.py rename to distarray/dist/functions.py index a4fe8568..1e4d5138 100644 --- a/distarray/functions.py +++ b/distarray/dist/functions.py @@ -8,10 +8,12 @@ Distributed unfuncs for distributed arrays. """ +from __future__ import absolute_import + import numpy from distarray.error import ContextError -from distarray.client import DistArray +from distarray.dist.distarray import DistArray __all__ = [] # unary_names and binary_names added to __all__ below. diff --git a/distarray/ipython_utils.py b/distarray/dist/ipython_utils.py similarity index 100% rename from distarray/ipython_utils.py rename to distarray/dist/ipython_utils.py diff --git a/distarray/client_map.py b/distarray/dist/maps.py similarity index 99% rename from distarray/client_map.py rename to distarray/dist/maps.py index f0abb16e..f5def126 100644 --- a/distarray/client_map.py +++ b/distarray/dist/maps.py @@ -21,6 +21,7 @@ `UnstructuredMap`. """ +from __future__ import absolute_import import operator from itertools import product diff --git a/distarray/random.py b/distarray/dist/random.py similarity index 98% rename from distarray/random.py rename to distarray/dist/random.py index 2eb101f8..245e213c 100644 --- a/distarray/random.py +++ b/distarray/dist/random.py @@ -7,9 +7,10 @@ """Emulate numpy.random""" +from __future__ import absolute_import -from distarray.client import DistArray -from distarray.client_map import Distribution +from distarray.dist.distarray import DistArray +from distarray.dist.maps import Distribution class Random(object): diff --git a/distarray/tests/__init__.py b/distarray/dist/tests/__init__.py similarity index 100% rename from distarray/tests/__init__.py rename to distarray/dist/tests/__init__.py diff --git a/distarray/tests/test_context.py b/distarray/dist/tests/test_context.py similarity index 97% rename from distarray/tests/test_context.py rename to distarray/dist/tests/test_context.py index 91c442e2..0fb3f72e 100644 --- a/distarray/tests/test_context.py +++ b/distarray/dist/tests/test_context.py @@ -17,8 +17,8 @@ import numpy -from distarray import Context -from distarray.ipython_utils import IPythonClient +from distarray.dist.context import Context +from distarray.dist.ipython_utils import IPythonClient from distarray.local import LocalArray diff --git a/distarray/tests/test_decorators.py b/distarray/dist/tests/test_decorators.py similarity index 98% rename from distarray/tests/test_decorators.py rename to distarray/dist/tests/test_decorators.py index b7357612..a372deab 100644 --- a/distarray/tests/test_decorators.py +++ b/distarray/dist/tests/test_decorators.py @@ -17,8 +17,8 @@ import numpy from numpy.testing import assert_array_equal -from distarray.context import Context -from distarray.decorators import DecoratorBase, local, vectorize +from distarray.dist.context import Context +from distarray.dist.decorators import DecoratorBase, local, vectorize from distarray.error import ContextError diff --git a/distarray/tests/test_client.py b/distarray/dist/tests/test_distarray.py similarity index 98% rename from distarray/tests/test_client.py rename to distarray/dist/tests/test_distarray.py index 38b0ff39..5ae4e529 100644 --- a/distarray/tests/test_client.py +++ b/distarray/dist/tests/test_distarray.py @@ -18,9 +18,9 @@ from numpy.testing import assert_array_equal, assert_allclose from distarray.externals.six.moves import range -from distarray.client import DistArray -from distarray.client_map import Distribution -from distarray.context import Context +from distarray.dist.distarray import DistArray +from distarray.dist.maps import Distribution +from distarray.dist.context import Context class TestDistArray(unittest.TestCase): diff --git a/distarray/tests/test_distributed_io.py b/distarray/dist/tests/test_distributed_io.py similarity index 98% rename from distarray/tests/test_distributed_io.py rename to distarray/dist/tests/test_distributed_io.py index bfa967c7..e7e053a7 100644 --- a/distarray/tests/test_distributed_io.py +++ b/distarray/dist/tests/test_distributed_io.py @@ -19,8 +19,8 @@ from distarray.externals.six.moves import range -from distarray.client import DistArray -from distarray.context import Context +from distarray.dist.distarray import DistArray +from distarray.dist.context import Context from distarray.testing import import_or_skip, temp_filepath diff --git a/distarray/tests/test_umath.py b/distarray/dist/tests/test_functions.py similarity index 96% rename from distarray/tests/test_umath.py rename to distarray/dist/tests/test_functions.py index b14d97f7..69c4109c 100644 --- a/distarray/tests/test_umath.py +++ b/distarray/dist/tests/test_functions.py @@ -16,8 +16,8 @@ import numpy as np from numpy.testing import assert_array_equal -import distarray -from distarray import Context +import distarray.dist.functions as functions +from distarray.dist.context import Context def add_checkers(cls, ops, checker_name): @@ -63,7 +63,7 @@ def check_binary_op(self, op_name): Check the two- and three-arg ufunc versions as well as the method version attached to a LocalArray. """ - op = getattr(distarray, op_name) + op = getattr(functions, op_name) ufunc = getattr(np, op_name) with warnings.catch_warnings(): # ignore inf, NaN warnings etc. @@ -78,7 +78,7 @@ def check_unary_op(self, op_name): Check the two- and three-arg ufunc versions as well as the method version attached to a LocalArray. """ - op = getattr(distarray, op_name) + op = getattr(functions, op_name) ufunc = getattr(np, op_name) with warnings.catch_warnings(): warnings.simplefilter("ignore", category=RuntimeWarning) diff --git a/distarray/tests/test_client_map.py b/distarray/dist/tests/test_maps.py similarity index 95% rename from distarray/tests/test_client_map.py rename to distarray/dist/tests/test_maps.py index 649b100a..dd55b0da 100644 --- a/distarray/tests/test_client_map.py +++ b/distarray/dist/tests/test_maps.py @@ -9,8 +9,8 @@ from distarray.externals.six.moves import range -from distarray import Context -from distarray import client_map +from distarray.dist.context import Context +from distarray.dist import maps as client_map class TestClientMap(unittest.TestCase): @@ -56,6 +56,3 @@ def test_2D_cc(self): rank = (r % nprocs_per_dim) * nprocs_per_dim + (c % nprocs_per_dim) actual = cm.owning_ranks((r,c)) self.assertSequenceEqual(actual, [rank]) - - - diff --git a/distarray/tests/test_random.py b/distarray/dist/tests/test_random.py similarity index 97% rename from distarray/tests/test_random.py rename to distarray/dist/tests/test_random.py index 4a94e737..1a9a24b6 100644 --- a/distarray/tests/test_random.py +++ b/distarray/dist/tests/test_random.py @@ -8,8 +8,8 @@ import unittest -from distarray.context import Context -from distarray.random import Random +from distarray.dist.context import Context +from distarray.dist.random import Random class TestRandom(unittest.TestCase): diff --git a/distarray/local/construct.py b/distarray/local/construct.py index 994fcd8c..5908fca5 100644 --- a/distarray/local/construct.py +++ b/distarray/local/construct.py @@ -6,9 +6,9 @@ from __future__ import division -from distarray.mpiutils import MPI +from distarray.local.mpiutils import MPI from distarray.local.error import NullCommError, InvalidBaseCommError -from distarray import mpiutils +from distarray.local import mpiutils # --------------------------------------------------------------------------- diff --git a/distarray/local/localarray.py b/distarray/local/localarray.py index 9386061c..b15f558d 100644 --- a/distarray/local/localarray.py +++ b/distarray/local/localarray.py @@ -19,7 +19,7 @@ from distarray.externals import six from distarray.externals.six.moves import zip -from distarray.mpiutils import MPI +from distarray.local.mpiutils import MPI from distarray.utils import _raise_nie from distarray.local import format, maps from distarray.local.error import InvalidDimensionError, IncompatibleArrayError diff --git a/distarray/mpiutils.py b/distarray/local/mpiutils.py similarity index 100% rename from distarray/mpiutils.py rename to distarray/local/mpiutils.py diff --git a/distarray/local/tests/paralleltest_mpiutils.py b/distarray/local/tests/paralleltest_mpiutils.py index 89e1e407..39483c41 100644 --- a/distarray/local/tests/paralleltest_mpiutils.py +++ b/distarray/local/tests/paralleltest_mpiutils.py @@ -11,7 +11,8 @@ from distarray.local.localarray import zeros from distarray.local.maps import Distribution from distarray.error import InvalidCommSizeError, InvalidRankError -from distarray.mpiutils import MPI, create_comm_of_size, create_comm_with_list +from distarray.local.mpiutils import (MPI, create_comm_of_size, + create_comm_with_list) class TestCreateCommAlternate(unittest.TestCase): diff --git a/distarray/tests/test_mpiutils.py b/distarray/local/tests/test_mpiutils.py similarity index 93% rename from distarray/tests/test_mpiutils.py rename to distarray/local/tests/test_mpiutils.py index 160f8fca..11512c3a 100644 --- a/distarray/tests/test_mpiutils.py +++ b/distarray/local/tests/test_mpiutils.py @@ -7,7 +7,7 @@ import unittest import numpy -from distarray.mpiutils import mpi_type_for_ndarray +from distarray.local.mpiutils import mpi_type_for_ndarray class TestMpiTypes(unittest.TestCase): diff --git a/distarray/testing.py b/distarray/testing.py index b66a1cab..d806cf4b 100644 --- a/distarray/testing.py +++ b/distarray/testing.py @@ -20,7 +20,7 @@ from distarray.externals import protocol_validator from distarray.error import InvalidCommSizeError -from distarray.mpiutils import MPI, create_comm_of_size +from distarray.local.mpiutils import MPI, create_comm_of_size def raise_typeerror(fn):