Split up client and local code.

distarray/__init__.py

@@ -5,7 +5,4 @@
 # ---------------------------------------------------------------------------
 
 __version__ = "0.3.0-dev"
-
-from distarray.client import DistArray
-from distarray.context import Context
-from distarray.functions import *
+DISTARRAY_BASE_NAME = '__distarray__'

distarray/apps/dacluster.py

@@ -18,7 +18,7 @@
 from subprocess import Popen, PIPE
 
 from distarray.externals import six
-from distarray.cleanup import clear_all
+from distarray.dist.cleanup import clear_all
 
 
 if six.PY2:

distarray/dist/__init__.py

@@ -0,0 +1,11 @@
+# encoding: utf-8
+# ---------------------------------------------------------------------------
+#  Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc.
+#  Distributed under the terms of the BSD License.  See COPYING.rst.
+# ---------------------------------------------------------------------------
+
+from __future__ import absolute_import
+
+from distarray.dist.distarray import DistArray
+from distarray.dist.context import Context
+from distarray.dist.functions import *

distarray/cleanup.py → distarray/dist/cleanup.py

@@ -4,9 +4,9 @@
 #  Distributed under the terms of the BSD License.  See COPYING.rst.
 # ---------------------------------------------------------------------------
 
-from __future__ import print_function
+from __future__ import print_function, absolute_import
 
-from distarray.ipython_utils import IPythonClient
+from distarray.dist.ipython_utils import IPythonClient
 
 def cleanup(view, module_name, prefix):
     """ Delete Context object with the given name from the given module"""
@@ -46,7 +46,7 @@ def get_keys_engine(prefix):
     for target, keys in keys_from_target.items():
         for key in keys:
             targets_from_key.setdefault(key, []).append(target)
-    
+
     return targets_from_key
 
 def clear(view):

distarray/context.py → distarray/dist/context.py

@@ -8,21 +8,21 @@
 communicate with localarrays.
 """
 
+from __future__ import absolute_import
 
 import uuid
 import collections
 import atexit
 
 import numpy
 
-from distarray import cleanup
+from distarray.dist import cleanup
 from distarray.externals import six
-from distarray.client import DistArray
-from distarray.client_map import Distribution
-from distarray.ipython_utils import IPythonClient
+from distarray.dist.distarray import DistArray
+from distarray.dist.maps import Distribution
 
-
-DISTARRAY_BASE_NAME = '__distarray__'
+from distarray.dist.ipython_utils import IPythonClient
+from distarray import DISTARRAY_BASE_NAME
 
 
 class Context(object):
@@ -70,7 +70,7 @@ def __init__(self, client=None, targets=None):
         #with self.view.sync_imports():
         #    import distarray
         self.view.execute("import distarray.local; "
-                          "import distarray.mpiutils; "
+                          "import distarray.local.mpiutils; "
                           "import numpy")
 
         self.context_key = self._setup_context_key()
@@ -91,13 +91,13 @@ def _setup_context_key(self):
 
     def _make_intracomm(self):
         def get_rank():
-            from distarray.mpiutils import COMM_PRIVATE
+            from distarray.local.mpiutils import COMM_PRIVATE
             return COMM_PRIVATE.Get_rank()
 
         # self.view's engines must encompass all ranks in the MPI communicator,
         # i.e., everything in rank_map.values().
         def get_size():
-            from distarray.mpiutils import COMM_PRIVATE
+            from distarray.local.mpiutils import COMM_PRIVATE
             return COMM_PRIVATE.Get_size()
 
         # get a mapping of IPython engine ID to MPI rank
@@ -112,7 +112,7 @@ def get_size():
         # MPI_Comm_create must be called on all engines, not just those
         # involved in the new communicator.
         comm_key = self._generate_key()
-        cmd = "%s = distarray.mpiutils.create_comm_with_list(%s)"
+        cmd = "%s = distarray.local.mpiutils.create_comm_with_list(%s)"
         cmd %= (comm_key, ranks)
         self.view.execute(cmd, block=True)
         return comm_key

distarray/decorators.py → distarray/dist/decorators.py

@@ -8,10 +8,12 @@
 Decorators for defining functions that use `DistArrays`.
 """
 
+from __future__ import absolute_import
+
 import functools
 
-from distarray.client import DistArray
-from distarray.context import Context
+from distarray.dist.distarray import DistArray
+from distarray.dist.context import Context
 from distarray.error import ContextError
 from distarray.utils import has_exactly_one
 

distarray/client.py → distarray/dist/distarray.py

@@ -12,13 +12,15 @@
 # Imports
 # ---------------------------------------------------------------------------
 
+from __future__ import absolute_import
+
 import operator
 from itertools import product
 
 import numpy as np
 
 import distarray
-from distarray.client_map import Distribution
+from distarray.dist.maps import Distribution
 from distarray.externals.six import next
 from distarray.utils import has_exactly_one, _raise_nie
 
@@ -351,113 +353,113 @@ def _rbinary_op_from_ufunc(self, other, func, lop_str, *args, **kwargs):
         return func(other, self, *args, **kwargs)
 
     def __add__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.add, '__radd__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.add, '__radd__', *args, **kwargs)
 
     def __sub__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.subtract, '__rsub__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.subtract, '__rsub__', *args, **kwargs)
 
     def __mul__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.multiply, '__rmul__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.multiply, '__rmul__', *args, **kwargs)
 
     def __div__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.divide, '__rdiv__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.divide, '__rdiv__', *args, **kwargs)
 
     def __truediv__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.true_divide, '__rtruediv__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.true_divide, '__rtruediv__', *args, **kwargs)
 
     def __floordiv__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.floor_divide, '__rfloordiv__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.floor_divide, '__rfloordiv__', *args, **kwargs)
 
     def __mod__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.mod, '__rdiv__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.mod, '__rdiv__', *args, **kwargs)
 
     def __pow__(self, other, modulo=None, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.power, '__rpower__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.power, '__rpower__', *args, **kwargs)
 
     def __lshift__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.left_shift, '__rlshift__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.left_shift, '__rlshift__', *args, **kwargs)
 
     def __rshift__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.right_shift, '__rrshift__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.right_shift, '__rrshift__', *args, **kwargs)
 
     def __and__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.bitwise_and, '__rand__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.bitwise_and, '__rand__', *args, **kwargs)
 
     def __or__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.bitwise_or, '__ror__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.bitwise_or, '__ror__', *args, **kwargs)
 
     def __xor__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.bitwise_xor, '__rxor__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.bitwise_xor, '__rxor__', *args, **kwargs)
 
     # Binary - right versions
 
     def __radd__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.add, '__add__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.add, '__add__', *args, **kwargs)
 
     def __rsub__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.subtract, '__sub__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.subtract, '__sub__', *args, **kwargs)
 
     def __rmul__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.multiply, '__mul__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.multiply, '__mul__', *args, **kwargs)
 
     def __rdiv__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.divide, '__div__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.divide, '__div__', *args, **kwargs)
 
     def __rtruediv__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.true_divide, '__truediv__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.true_divide, '__truediv__', *args, **kwargs)
 
     def __rfloordiv__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.floor_divide, '__floordiv__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.floor_divide, '__floordiv__', *args, **kwargs)
 
     def __rmod__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.mod, '__mod__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.mod, '__mod__', *args, **kwargs)
 
     def __rpow__(self, other, modulo=None, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.power, '__pow__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.power, '__pow__', *args, **kwargs)
 
     def __rlshift__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.left_shift, '__lshift__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.left_shift, '__lshift__', *args, **kwargs)
 
     def __rrshift__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.right_shift, '__rshift__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.right_shift, '__rshift__', *args, **kwargs)
 
     def __rand__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.bitwise_and, '__and__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.bitwise_and, '__and__', *args, **kwargs)
 
     def __ror__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.bitwise_or, '__or__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.bitwise_or, '__or__', *args, **kwargs)
 
     def __rxor__(self, other, *args, **kwargs):
-        return self._rbinary_op_from_ufunc(other, distarray.bitwise_xor, '__xor__', *args, **kwargs)
+        return self._rbinary_op_from_ufunc(other, distarray.dist.bitwise_xor, '__xor__', *args, **kwargs)
 
     def __neg__(self, *args, **kwargs):
-        return distarray.negative(self, *args, **kwargs)
+        return distarray.dist.negative(self, *args, **kwargs)
 
     def __pos__(self, *args, **kwargs):
         return self
 
     def __abs__(self, *args, **kwargs):
-        return distarray.abs(self, *args, **kwargs)
+        return distarray.dist.abs(self, *args, **kwargs)
 
     def __invert__(self, *args, **kwargs):
-        return distarray.invert(self, *args, **kwargs)
+        return distarray.dist.invert(self, *args, **kwargs)
 
     # Boolean comparisons
 
     def __lt__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.less, '__lt__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.less, '__lt__', *args, **kwargs)
 
     def __le__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.less_equal, '__le__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.less_equal, '__le__', *args, **kwargs)
 
     def __eq__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.equal, '__eq__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.equal, '__eq__', *args, **kwargs)
 
     def __ne__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.not_equal, '__ne__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.not_equal, '__ne__', *args, **kwargs)
 
     def __gt__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.greater, '__gt__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.greater, '__gt__', *args, **kwargs)
 
     def __ge__(self, other, *args, **kwargs):
-        return self._binary_op_from_ufunc(other, distarray.greater_equal, '__ge__', *args, **kwargs)
+        return self._binary_op_from_ufunc(other, distarray.dist.greater_equal, '__ge__', *args, **kwargs)

distarray/functions.py → distarray/dist/functions.py

@@ -8,10 +8,12 @@
 Distributed unfuncs for distributed arrays.
 """
 
+from __future__ import absolute_import
+
 import numpy
 
 from distarray.error import ContextError
-from distarray.client import DistArray
+from distarray.dist.distarray import DistArray
 
 
 __all__ = []  # unary_names and binary_names added to __all__ below.

distarray/client_map.py → distarray/dist/maps.py

@@ -21,6 +21,7 @@
 `UnstructuredMap`.
 
 """
+from __future__ import absolute_import
 
 import operator
 from itertools import product

distarray/random.py → distarray/dist/random.py

@@ -7,9 +7,10 @@
 
 """Emulate numpy.random"""
 
+from __future__ import absolute_import
 
-from distarray.client import DistArray
-from distarray.client_map import Distribution
+from distarray.dist.distarray import DistArray
+from distarray.dist.maps import Distribution
 
 
 class Random(object):

distarray/tests/test_context.py → distarray/dist/tests/test_context.py

@@ -17,8 +17,8 @@
 
 import numpy
 
-from distarray import Context
-from distarray.ipython_utils import IPythonClient
+from distarray.dist.context import Context
+from distarray.dist.ipython_utils import IPythonClient
 from distarray.local import LocalArray
 
 

distarray/tests/test_decorators.py → distarray/dist/tests/test_decorators.py

@@ -17,8 +17,8 @@
 import numpy
 from numpy.testing import assert_array_equal
 
-from distarray.context import Context
-from distarray.decorators import DecoratorBase, local, vectorize
+from distarray.dist.context import Context
+from distarray.dist.decorators import DecoratorBase, local, vectorize
 from distarray.error import ContextError
 
 

distarray/tests/test_client.py → distarray/dist/tests/test_distarray.py

@@ -18,9 +18,9 @@
 from numpy.testing import assert_array_equal, assert_allclose
 
 from distarray.externals.six.moves import range
-from distarray.client import DistArray
-from distarray.client_map import Distribution
-from distarray.context import Context
+from distarray.dist.distarray import DistArray
+from distarray.dist.maps import Distribution
+from distarray.dist.context import Context
 
 
 class TestDistArray(unittest.TestCase):

distarray/tests/test_distributed_io.py → distarray/dist/tests/test_distributed_io.py

@@ -19,8 +19,8 @@
 
 from distarray.externals.six.moves import range
 
-from distarray.client import DistArray
-from distarray.context import Context
+from distarray.dist.distarray import DistArray
+from distarray.dist.context import Context
 from distarray.testing import import_or_skip, temp_filepath