From 4b8cfa5395735e96b7240fba90781611cae2f8c0 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 21 Apr 2014 10:19:37 -0500 Subject: [PATCH 01/11] Put client side modules in distarray/client/ and change imports. --- distarray/__init__.py | 4 -- distarray/client/__init__.py | 9 ++++ distarray/{ => client}/client.py | 74 +++++++++++++------------- distarray/{ => client}/client_map.py | 0 distarray/{ => client}/context.py | 5 +- distarray/{ => client}/decorators.py | 4 +- distarray/{ => client}/functions.py | 2 +- distarray/{ => client}/random.py | 5 +- distarray/tests/test_client.py | 6 +-- distarray/tests/test_client_map.py | 4 +- distarray/tests/test_context.py | 2 +- distarray/tests/test_decorators.py | 4 +- distarray/tests/test_distributed_io.py | 4 +- distarray/tests/test_random.py | 4 +- distarray/tests/test_umath.py | 8 +-- 15 files changed, 70 insertions(+), 65 deletions(-) create mode 100644 distarray/client/__init__.py rename distarray/{ => client}/client.py (81%) rename distarray/{ => client}/client_map.py (100%) rename distarray/{ => client}/context.py (99%) rename distarray/{ => client}/decorators.py (98%) rename distarray/{ => client}/functions.py (98%) rename distarray/{ => client}/random.py (99%) diff --git a/distarray/__init__.py b/distarray/__init__.py index 4dadf681..2b2f592a 100644 --- a/distarray/__init__.py +++ b/distarray/__init__.py @@ -5,7 +5,3 @@ # --------------------------------------------------------------------------- __version__ = "0.3.0-dev" - -from distarray.client import DistArray -from distarray.context import Context -from distarray.functions import * diff --git a/distarray/client/__init__.py b/distarray/client/__init__.py new file mode 100644 index 00000000..0fdac40d --- /dev/null +++ b/distarray/client/__init__.py @@ -0,0 +1,9 @@ +# encoding: utf-8 +# --------------------------------------------------------------------------- +# Copyright (C) 2008-2014, IPython Development Team and Enthought, Inc. +# Distributed under the terms of the BSD License. See COPYING.rst. +# --------------------------------------------------------------------------- + +from distarray.client.client import DistArray +from distarray.client.context import Context +from distarray.client.functions import * diff --git a/distarray/client.py b/distarray/client/client.py similarity index 81% rename from distarray/client.py rename to distarray/client/client.py index 0276285a..0214af26 100644 --- a/distarray/client.py +++ b/distarray/client/client.py @@ -17,8 +17,8 @@ import numpy as np -import distarray -from distarray.client_map import Distribution +from distarray.client.client_map import Distribution +import distarray.client.functions as dafuncs from distarray.externals.six import next from distarray.utils import has_exactly_one, _raise_nie @@ -351,113 +351,113 @@ def _rbinary_op_from_ufunc(self, other, func, lop_str, *args, **kwargs): return func(other, self, *args, **kwargs) def __add__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.add, '__radd__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.add, '__radd__', *args, **kwargs) def __sub__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.subtract, '__rsub__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.subtract, '__rsub__', *args, **kwargs) def __mul__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.multiply, '__rmul__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.multiply, '__rmul__', *args, **kwargs) def __div__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.divide, '__rdiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.divide, '__rdiv__', *args, **kwargs) def __truediv__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.true_divide, '__rtruediv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.true_divide, '__rtruediv__', *args, **kwargs) def __floordiv__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.floor_divide, '__rfloordiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.floor_divide, '__rfloordiv__', *args, **kwargs) def __mod__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.mod, '__rdiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.mod, '__rdiv__', *args, **kwargs) def __pow__(self, other, modulo=None, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.power, '__rpower__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.power, '__rpower__', *args, **kwargs) def __lshift__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.left_shift, '__rlshift__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.left_shift, '__rlshift__', *args, **kwargs) def __rshift__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.right_shift, '__rrshift__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.right_shift, '__rrshift__', *args, **kwargs) def __and__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.bitwise_and, '__rand__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.bitwise_and, '__rand__', *args, **kwargs) def __or__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.bitwise_or, '__ror__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.bitwise_or, '__ror__', *args, **kwargs) def __xor__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.bitwise_xor, '__rxor__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.bitwise_xor, '__rxor__', *args, **kwargs) # Binary - right versions def __radd__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.add, '__add__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.add, '__add__', *args, **kwargs) def __rsub__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.subtract, '__sub__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.subtract, '__sub__', *args, **kwargs) def __rmul__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.multiply, '__mul__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.multiply, '__mul__', *args, **kwargs) def __rdiv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.divide, '__div__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.divide, '__div__', *args, **kwargs) def __rtruediv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.true_divide, '__truediv__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.true_divide, '__truediv__', *args, **kwargs) def __rfloordiv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.floor_divide, '__floordiv__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.floor_divide, '__floordiv__', *args, **kwargs) def __rmod__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.mod, '__mod__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.mod, '__mod__', *args, **kwargs) def __rpow__(self, other, modulo=None, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.power, '__pow__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.power, '__pow__', *args, **kwargs) def __rlshift__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.left_shift, '__lshift__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.left_shift, '__lshift__', *args, **kwargs) def __rrshift__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.right_shift, '__rshift__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.right_shift, '__rshift__', *args, **kwargs) def __rand__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.bitwise_and, '__and__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.bitwise_and, '__and__', *args, **kwargs) def __ror__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.bitwise_or, '__or__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.bitwise_or, '__or__', *args, **kwargs) def __rxor__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.bitwise_xor, '__xor__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, dafuncs.bitwise_xor, '__xor__', *args, **kwargs) def __neg__(self, *args, **kwargs): - return distarray.negative(self, *args, **kwargs) + return dafuncs.negative(self, *args, **kwargs) def __pos__(self, *args, **kwargs): return self def __abs__(self, *args, **kwargs): - return distarray.abs(self, *args, **kwargs) + return dafuncs.abs(self, *args, **kwargs) def __invert__(self, *args, **kwargs): - return distarray.invert(self, *args, **kwargs) + return dafuncs.invert(self, *args, **kwargs) # Boolean comparisons def __lt__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.less, '__lt__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.less, '__lt__', *args, **kwargs) def __le__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.less_equal, '__le__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.less_equal, '__le__', *args, **kwargs) def __eq__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.equal, '__eq__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.equal, '__eq__', *args, **kwargs) def __ne__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.not_equal, '__ne__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.not_equal, '__ne__', *args, **kwargs) def __gt__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.greater, '__gt__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.greater, '__gt__', *args, **kwargs) def __ge__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.greater_equal, '__ge__', *args, **kwargs) + return self._binary_op_from_ufunc(other, dafuncs.greater_equal, '__ge__', *args, **kwargs) diff --git a/distarray/client_map.py b/distarray/client/client_map.py similarity index 100% rename from distarray/client_map.py rename to distarray/client/client_map.py diff --git a/distarray/context.py b/distarray/client/context.py similarity index 99% rename from distarray/context.py rename to distarray/client/context.py index 74434670..e60aecb0 100644 --- a/distarray/context.py +++ b/distarray/client/context.py @@ -17,8 +17,9 @@ from distarray import cleanup from distarray.externals import six -from distarray.client import DistArray -from distarray.client_map import Distribution +from distarray.client.client import DistArray +from distarray.client.client_map import Distribution + from distarray.ipython_utils import IPythonClient diff --git a/distarray/decorators.py b/distarray/client/decorators.py similarity index 98% rename from distarray/decorators.py rename to distarray/client/decorators.py index 3dd60150..cf1f265b 100644 --- a/distarray/decorators.py +++ b/distarray/client/decorators.py @@ -10,8 +10,8 @@ import functools -from distarray.client import DistArray -from distarray.context import Context +from distarray.client.client import DistArray +from distarray.client.context import Context from distarray.error import ContextError from distarray.utils import has_exactly_one diff --git a/distarray/functions.py b/distarray/client/functions.py similarity index 98% rename from distarray/functions.py rename to distarray/client/functions.py index a4fe8568..2e489f58 100644 --- a/distarray/functions.py +++ b/distarray/client/functions.py @@ -11,7 +11,7 @@ import numpy from distarray.error import ContextError -from distarray.client import DistArray +from distarray.client.client import DistArray __all__ = [] # unary_names and binary_names added to __all__ below. diff --git a/distarray/random.py b/distarray/client/random.py similarity index 99% rename from distarray/random.py rename to distarray/client/random.py index 2eb101f8..655cac71 100644 --- a/distarray/random.py +++ b/distarray/client/random.py @@ -7,9 +7,8 @@ """Emulate numpy.random""" - -from distarray.client import DistArray -from distarray.client_map import Distribution +from distarray.client.client import DistArray +from distarray.client.client_map import Distribution class Random(object): diff --git a/distarray/tests/test_client.py b/distarray/tests/test_client.py index 38b0ff39..c8b87a04 100644 --- a/distarray/tests/test_client.py +++ b/distarray/tests/test_client.py @@ -18,9 +18,9 @@ from numpy.testing import assert_array_equal, assert_allclose from distarray.externals.six.moves import range -from distarray.client import DistArray -from distarray.client_map import Distribution -from distarray.context import Context +from distarray.client.client import DistArray +from distarray.client.client_map import Distribution +from distarray.client.context import Context class TestDistArray(unittest.TestCase): diff --git a/distarray/tests/test_client_map.py b/distarray/tests/test_client_map.py index 649b100a..cdd89db0 100644 --- a/distarray/tests/test_client_map.py +++ b/distarray/tests/test_client_map.py @@ -9,8 +9,8 @@ from distarray.externals.six.moves import range -from distarray import Context -from distarray import client_map +from distarray.client.context import Context +from distarray.client import client_map class TestClientMap(unittest.TestCase): diff --git a/distarray/tests/test_context.py b/distarray/tests/test_context.py index 91c442e2..07b5dd01 100644 --- a/distarray/tests/test_context.py +++ b/distarray/tests/test_context.py @@ -17,7 +17,7 @@ import numpy -from distarray import Context +from distarray.client.context import Context from distarray.ipython_utils import IPythonClient from distarray.local import LocalArray diff --git a/distarray/tests/test_decorators.py b/distarray/tests/test_decorators.py index b7357612..964b292a 100644 --- a/distarray/tests/test_decorators.py +++ b/distarray/tests/test_decorators.py @@ -17,8 +17,8 @@ import numpy from numpy.testing import assert_array_equal -from distarray.context import Context -from distarray.decorators import DecoratorBase, local, vectorize +from distarray.client.context import Context +from distarray.client.decorators import DecoratorBase, local, vectorize from distarray.error import ContextError diff --git a/distarray/tests/test_distributed_io.py b/distarray/tests/test_distributed_io.py index bfa967c7..dca6d941 100644 --- a/distarray/tests/test_distributed_io.py +++ b/distarray/tests/test_distributed_io.py @@ -19,8 +19,8 @@ from distarray.externals.six.moves import range -from distarray.client import DistArray -from distarray.context import Context +from distarray.client.client import DistArray +from distarray.client.context import Context from distarray.testing import import_or_skip, temp_filepath diff --git a/distarray/tests/test_random.py b/distarray/tests/test_random.py index 4a94e737..9d610e1d 100644 --- a/distarray/tests/test_random.py +++ b/distarray/tests/test_random.py @@ -8,8 +8,8 @@ import unittest -from distarray.context import Context -from distarray.random import Random +from distarray.client.context import Context +from distarray.client.random import Random class TestRandom(unittest.TestCase): diff --git a/distarray/tests/test_umath.py b/distarray/tests/test_umath.py index b14d97f7..7b0922c8 100644 --- a/distarray/tests/test_umath.py +++ b/distarray/tests/test_umath.py @@ -16,8 +16,8 @@ import numpy as np from numpy.testing import assert_array_equal -import distarray -from distarray import Context +import distarray.client.functions as functions +from distarray.client.context import Context def add_checkers(cls, ops, checker_name): @@ -63,7 +63,7 @@ def check_binary_op(self, op_name): Check the two- and three-arg ufunc versions as well as the method version attached to a LocalArray. """ - op = getattr(distarray, op_name) + op = getattr(functions, op_name) ufunc = getattr(np, op_name) with warnings.catch_warnings(): # ignore inf, NaN warnings etc. @@ -78,7 +78,7 @@ def check_unary_op(self, op_name): Check the two- and three-arg ufunc versions as well as the method version attached to a LocalArray. """ - op = getattr(distarray, op_name) + op = getattr(functions, op_name) ufunc = getattr(np, op_name) with warnings.catch_warnings(): warnings.simplefilter("ignore", category=RuntimeWarning) From c2db971d85b194339da187915af58fd104281c20 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 21 Apr 2014 11:54:06 -0500 Subject: [PATCH 02/11] move client.py -> distarray.py and change imports to reflect this. --- distarray/client/__init__.py | 2 +- distarray/client/context.py | 2 +- distarray/client/decorators.py | 2 +- distarray/client/{client.py => distarray.py} | 0 distarray/client/functions.py | 2 +- distarray/client/random.py | 2 +- distarray/tests/test_client.py | 2 +- distarray/tests/test_distributed_io.py | 2 +- 8 files changed, 7 insertions(+), 7 deletions(-) rename distarray/client/{client.py => distarray.py} (100%) diff --git a/distarray/client/__init__.py b/distarray/client/__init__.py index 0fdac40d..715ae47b 100644 --- a/distarray/client/__init__.py +++ b/distarray/client/__init__.py @@ -4,6 +4,6 @@ # Distributed under the terms of the BSD License. See COPYING.rst. # --------------------------------------------------------------------------- -from distarray.client.client import DistArray +from distarray.client.distarray import DistArray from distarray.client.context import Context from distarray.client.functions import * diff --git a/distarray/client/context.py b/distarray/client/context.py index e60aecb0..e637fe8d 100644 --- a/distarray/client/context.py +++ b/distarray/client/context.py @@ -17,7 +17,7 @@ from distarray import cleanup from distarray.externals import six -from distarray.client.client import DistArray +from distarray.client.distarray import DistArray from distarray.client.client_map import Distribution from distarray.ipython_utils import IPythonClient diff --git a/distarray/client/decorators.py b/distarray/client/decorators.py index cf1f265b..d25fed8a 100644 --- a/distarray/client/decorators.py +++ b/distarray/client/decorators.py @@ -10,7 +10,7 @@ import functools -from distarray.client.client import DistArray +from distarray.client.distarray import DistArray from distarray.client.context import Context from distarray.error import ContextError from distarray.utils import has_exactly_one diff --git a/distarray/client/client.py b/distarray/client/distarray.py similarity index 100% rename from distarray/client/client.py rename to distarray/client/distarray.py diff --git a/distarray/client/functions.py b/distarray/client/functions.py index 2e489f58..b3ff520d 100644 --- a/distarray/client/functions.py +++ b/distarray/client/functions.py @@ -11,7 +11,7 @@ import numpy from distarray.error import ContextError -from distarray.client.client import DistArray +from distarray.client.distarray import DistArray __all__ = [] # unary_names and binary_names added to __all__ below. diff --git a/distarray/client/random.py b/distarray/client/random.py index 655cac71..4931826c 100644 --- a/distarray/client/random.py +++ b/distarray/client/random.py @@ -7,7 +7,7 @@ """Emulate numpy.random""" -from distarray.client.client import DistArray +from distarray.client.distarray import DistArray from distarray.client.client_map import Distribution diff --git a/distarray/tests/test_client.py b/distarray/tests/test_client.py index c8b87a04..fa296c97 100644 --- a/distarray/tests/test_client.py +++ b/distarray/tests/test_client.py @@ -18,7 +18,7 @@ from numpy.testing import assert_array_equal, assert_allclose from distarray.externals.six.moves import range -from distarray.client.client import DistArray +from distarray.client.distarray import DistArray from distarray.client.client_map import Distribution from distarray.client.context import Context diff --git a/distarray/tests/test_distributed_io.py b/distarray/tests/test_distributed_io.py index dca6d941..4c0c6dcd 100644 --- a/distarray/tests/test_distributed_io.py +++ b/distarray/tests/test_distributed_io.py @@ -19,7 +19,7 @@ from distarray.externals.six.moves import range -from distarray.client.client import DistArray +from distarray.client.distarray import DistArray from distarray.client.context import Context from distarray.testing import import_or_skip, temp_filepath From 421d07dedb87c2e1ef160ff76ea0442c6fb46316 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 21 Apr 2014 11:59:59 -0500 Subject: [PATCH 03/11] move DISTARRAY_BASE_NAME into top level __init__.py This is because dacluster should not have to impor distarray/client/ to clean things up. --- distarray/__init__.py | 1 + distarray/client/context.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/distarray/__init__.py b/distarray/__init__.py index 2b2f592a..4c53159f 100644 --- a/distarray/__init__.py +++ b/distarray/__init__.py @@ -5,3 +5,4 @@ # --------------------------------------------------------------------------- __version__ = "0.3.0-dev" +DISTARRAY_BASE_NAME = '__distarray__' diff --git a/distarray/client/context.py b/distarray/client/context.py index e637fe8d..0ef26b07 100644 --- a/distarray/client/context.py +++ b/distarray/client/context.py @@ -21,9 +21,9 @@ from distarray.client.client_map import Distribution from distarray.ipython_utils import IPythonClient +from distarray import DISTARRAY_BASE_NAME - -DISTARRAY_BASE_NAME = '__distarray__' +atexit.register(cleanup.cleanup_all, DISTARRAY_BASE_NAME) class Context(object): From 11bd41ecb06834f69972ac61dfd140259334f20a Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 21 Apr 2014 12:14:39 -0500 Subject: [PATCH 04/11] Fix circular import issue with distarray.py --- distarray/client/distarray.py | 73 ++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/distarray/client/distarray.py b/distarray/client/distarray.py index 0214af26..4b022430 100644 --- a/distarray/client/distarray.py +++ b/distarray/client/distarray.py @@ -12,13 +12,14 @@ # Imports # --------------------------------------------------------------------------- +print(__name__) import operator from itertools import product import numpy as np +import distarray from distarray.client.client_map import Distribution -import distarray.client.functions as dafuncs from distarray.externals.six import next from distarray.utils import has_exactly_one, _raise_nie @@ -351,113 +352,113 @@ def _rbinary_op_from_ufunc(self, other, func, lop_str, *args, **kwargs): return func(other, self, *args, **kwargs) def __add__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.add, '__radd__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.add, '__radd__', *args, **kwargs) def __sub__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.subtract, '__rsub__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.subtract, '__rsub__', *args, **kwargs) def __mul__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.multiply, '__rmul__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.multiply, '__rmul__', *args, **kwargs) def __div__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.divide, '__rdiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.divide, '__rdiv__', *args, **kwargs) def __truediv__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.true_divide, '__rtruediv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.true_divide, '__rtruediv__', *args, **kwargs) def __floordiv__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.floor_divide, '__rfloordiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.floor_divide, '__rfloordiv__', *args, **kwargs) def __mod__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.mod, '__rdiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.mod, '__rdiv__', *args, **kwargs) def __pow__(self, other, modulo=None, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.power, '__rpower__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.power, '__rpower__', *args, **kwargs) def __lshift__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.left_shift, '__rlshift__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.left_shift, '__rlshift__', *args, **kwargs) def __rshift__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.right_shift, '__rrshift__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.right_shift, '__rrshift__', *args, **kwargs) def __and__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.bitwise_and, '__rand__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.bitwise_and, '__rand__', *args, **kwargs) def __or__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.bitwise_or, '__ror__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.bitwise_or, '__ror__', *args, **kwargs) def __xor__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.bitwise_xor, '__rxor__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.bitwise_xor, '__rxor__', *args, **kwargs) # Binary - right versions def __radd__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.add, '__add__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.add, '__add__', *args, **kwargs) def __rsub__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.subtract, '__sub__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.subtract, '__sub__', *args, **kwargs) def __rmul__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.multiply, '__mul__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.multiply, '__mul__', *args, **kwargs) def __rdiv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.divide, '__div__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.divide, '__div__', *args, **kwargs) def __rtruediv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.true_divide, '__truediv__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.true_divide, '__truediv__', *args, **kwargs) def __rfloordiv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.floor_divide, '__floordiv__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.floor_divide, '__floordiv__', *args, **kwargs) def __rmod__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.mod, '__mod__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.mod, '__mod__', *args, **kwargs) def __rpow__(self, other, modulo=None, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.power, '__pow__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.power, '__pow__', *args, **kwargs) def __rlshift__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.left_shift, '__lshift__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.left_shift, '__lshift__', *args, **kwargs) def __rrshift__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.right_shift, '__rshift__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.right_shift, '__rshift__', *args, **kwargs) def __rand__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.bitwise_and, '__and__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.bitwise_and, '__and__', *args, **kwargs) def __ror__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.bitwise_or, '__or__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.bitwise_or, '__or__', *args, **kwargs) def __rxor__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, dafuncs.bitwise_xor, '__xor__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.client.bitwise_xor, '__xor__', *args, **kwargs) def __neg__(self, *args, **kwargs): - return dafuncs.negative(self, *args, **kwargs) + return distarray.client.negative(self, *args, **kwargs) def __pos__(self, *args, **kwargs): return self def __abs__(self, *args, **kwargs): - return dafuncs.abs(self, *args, **kwargs) + return distarray.client.abs(self, *args, **kwargs) def __invert__(self, *args, **kwargs): - return dafuncs.invert(self, *args, **kwargs) + return distarray.client.invert(self, *args, **kwargs) # Boolean comparisons def __lt__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.less, '__lt__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.less, '__lt__', *args, **kwargs) def __le__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.less_equal, '__le__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.less_equal, '__le__', *args, **kwargs) def __eq__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.equal, '__eq__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.equal, '__eq__', *args, **kwargs) def __ne__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.not_equal, '__ne__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.not_equal, '__ne__', *args, **kwargs) def __gt__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.greater, '__gt__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.greater, '__gt__', *args, **kwargs) def __ge__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, dafuncs.greater_equal, '__ge__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.client.greater_equal, '__ge__', *args, **kwargs) From 047f9e3824fd70ba91676c1414fd21eb508db550 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Mon, 21 Apr 2014 15:03:32 -0500 Subject: [PATCH 05/11] client_maps.py -> maps.py. add __future__.absolute_import where needed. Since the module distarray/client/distarray.py's name conflicts with the package name we import `absolute_import` everywhere in the file. --- distarray/client/__init__.py | 2 ++ distarray/client/context.py | 3 ++- distarray/client/decorators.py | 2 ++ distarray/client/distarray.py | 5 +++-- distarray/client/functions.py | 2 ++ distarray/client/{client_map.py => maps.py} | 1 + distarray/client/random.py | 2 ++ distarray/tests/test_client.py | 2 +- distarray/tests/test_client_map.py | 2 +- 9 files changed, 16 insertions(+), 5 deletions(-) rename distarray/client/{client_map.py => maps.py} (99%) diff --git a/distarray/client/__init__.py b/distarray/client/__init__.py index 715ae47b..d30f07b1 100644 --- a/distarray/client/__init__.py +++ b/distarray/client/__init__.py @@ -4,6 +4,8 @@ # Distributed under the terms of the BSD License. See COPYING.rst. # --------------------------------------------------------------------------- +from __future__ import absolute_import + from distarray.client.distarray import DistArray from distarray.client.context import Context from distarray.client.functions import * diff --git a/distarray/client/context.py b/distarray/client/context.py index 0ef26b07..3b6b3ecb 100644 --- a/distarray/client/context.py +++ b/distarray/client/context.py @@ -8,6 +8,7 @@ communicate with localarrays. """ +from __future__ import absolute_import import uuid import collections @@ -18,7 +19,7 @@ from distarray import cleanup from distarray.externals import six from distarray.client.distarray import DistArray -from distarray.client.client_map import Distribution +from distarray.client.maps import Distribution from distarray.ipython_utils import IPythonClient from distarray import DISTARRAY_BASE_NAME diff --git a/distarray/client/decorators.py b/distarray/client/decorators.py index d25fed8a..1a328867 100644 --- a/distarray/client/decorators.py +++ b/distarray/client/decorators.py @@ -8,6 +8,8 @@ Decorators for defining functions that use `DistArrays`. """ +from __future__ import absolute_import + import functools from distarray.client.distarray import DistArray diff --git a/distarray/client/distarray.py b/distarray/client/distarray.py index 4b022430..252b020b 100644 --- a/distarray/client/distarray.py +++ b/distarray/client/distarray.py @@ -12,14 +12,15 @@ # Imports # --------------------------------------------------------------------------- -print(__name__) +from __future__ import absolute_import + import operator from itertools import product import numpy as np import distarray -from distarray.client.client_map import Distribution +from distarray.client.maps import Distribution from distarray.externals.six import next from distarray.utils import has_exactly_one, _raise_nie diff --git a/distarray/client/functions.py b/distarray/client/functions.py index b3ff520d..4b6b44d1 100644 --- a/distarray/client/functions.py +++ b/distarray/client/functions.py @@ -8,6 +8,8 @@ Distributed unfuncs for distributed arrays. """ +from __future__ import absolute_import + import numpy from distarray.error import ContextError diff --git a/distarray/client/client_map.py b/distarray/client/maps.py similarity index 99% rename from distarray/client/client_map.py rename to distarray/client/maps.py index f0abb16e..f5def126 100644 --- a/distarray/client/client_map.py +++ b/distarray/client/maps.py @@ -21,6 +21,7 @@ `UnstructuredMap`. """ +from __future__ import absolute_import import operator from itertools import product diff --git a/distarray/client/random.py b/distarray/client/random.py index 4931826c..5083d93b 100644 --- a/distarray/client/random.py +++ b/distarray/client/random.py @@ -7,6 +7,8 @@ """Emulate numpy.random""" +from __future__ import absolute_import + from distarray.client.distarray import DistArray from distarray.client.client_map import Distribution diff --git a/distarray/tests/test_client.py b/distarray/tests/test_client.py index fa296c97..eb83b993 100644 --- a/distarray/tests/test_client.py +++ b/distarray/tests/test_client.py @@ -19,7 +19,7 @@ from distarray.externals.six.moves import range from distarray.client.distarray import DistArray -from distarray.client.client_map import Distribution +from distarray.client.maps import Distribution from distarray.client.context import Context diff --git a/distarray/tests/test_client_map.py b/distarray/tests/test_client_map.py index cdd89db0..949ffc46 100644 --- a/distarray/tests/test_client_map.py +++ b/distarray/tests/test_client_map.py @@ -10,7 +10,7 @@ from distarray.externals.six.moves import range from distarray.client.context import Context -from distarray.client import client_map +from distarray.client import maps as client_map class TestClientMap(unittest.TestCase): From 0fade1038930328a4b739e8c4efe3569a4087d24 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 25 Apr 2014 10:01:05 -0500 Subject: [PATCH 06/11] rename client directory to dist --- distarray/{client => dist}/__init__.py | 6 +- distarray/{client => dist}/context.py | 4 +- distarray/{client => dist}/decorators.py | 4 +- distarray/{client => dist}/distarray.py | 72 ++++++++++++------------ distarray/{client => dist}/functions.py | 2 +- distarray/{client => dist}/maps.py | 0 distarray/{client => dist}/random.py | 4 +- distarray/tests/test_client.py | 6 +- distarray/tests/test_client_map.py | 7 +-- distarray/tests/test_context.py | 2 +- distarray/tests/test_decorators.py | 4 +- distarray/tests/test_distributed_io.py | 4 +- distarray/tests/test_random.py | 4 +- distarray/tests/test_umath.py | 4 +- 14 files changed, 60 insertions(+), 63 deletions(-) rename distarray/{client => dist}/__init__.py (72%) rename distarray/{client => dist}/context.py (99%) rename distarray/{client => dist}/decorators.py (98%) rename distarray/{client => dist}/distarray.py (80%) rename distarray/{client => dist}/functions.py (98%) rename distarray/{client => dist}/maps.py (100%) rename distarray/{client => dist}/random.py (99%) diff --git a/distarray/client/__init__.py b/distarray/dist/__init__.py similarity index 72% rename from distarray/client/__init__.py rename to distarray/dist/__init__.py index d30f07b1..8590522a 100644 --- a/distarray/client/__init__.py +++ b/distarray/dist/__init__.py @@ -6,6 +6,6 @@ from __future__ import absolute_import -from distarray.client.distarray import DistArray -from distarray.client.context import Context -from distarray.client.functions import * +from distarray.dist.distarray import DistArray +from distarray.dist.context import Context +from distarray.dist.functions import * diff --git a/distarray/client/context.py b/distarray/dist/context.py similarity index 99% rename from distarray/client/context.py rename to distarray/dist/context.py index 3b6b3ecb..be346f4d 100644 --- a/distarray/client/context.py +++ b/distarray/dist/context.py @@ -18,8 +18,8 @@ from distarray import cleanup from distarray.externals import six -from distarray.client.distarray import DistArray -from distarray.client.maps import Distribution +from distarray.dist.distarray import DistArray +from distarray.dist.maps import Distribution from distarray.ipython_utils import IPythonClient from distarray import DISTARRAY_BASE_NAME diff --git a/distarray/client/decorators.py b/distarray/dist/decorators.py similarity index 98% rename from distarray/client/decorators.py rename to distarray/dist/decorators.py index 1a328867..37427807 100644 --- a/distarray/client/decorators.py +++ b/distarray/dist/decorators.py @@ -12,8 +12,8 @@ import functools -from distarray.client.distarray import DistArray -from distarray.client.context import Context +from distarray.dist.distarray import DistArray +from distarray.dist.context import Context from distarray.error import ContextError from distarray.utils import has_exactly_one diff --git a/distarray/client/distarray.py b/distarray/dist/distarray.py similarity index 80% rename from distarray/client/distarray.py rename to distarray/dist/distarray.py index 252b020b..aa87e7c3 100644 --- a/distarray/client/distarray.py +++ b/distarray/dist/distarray.py @@ -20,7 +20,7 @@ import numpy as np import distarray -from distarray.client.maps import Distribution +from distarray.dist.maps import Distribution from distarray.externals.six import next from distarray.utils import has_exactly_one, _raise_nie @@ -353,113 +353,113 @@ def _rbinary_op_from_ufunc(self, other, func, lop_str, *args, **kwargs): return func(other, self, *args, **kwargs) def __add__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.add, '__radd__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.add, '__radd__', *args, **kwargs) def __sub__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.subtract, '__rsub__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.subtract, '__rsub__', *args, **kwargs) def __mul__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.multiply, '__rmul__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.multiply, '__rmul__', *args, **kwargs) def __div__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.divide, '__rdiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.divide, '__rdiv__', *args, **kwargs) def __truediv__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.true_divide, '__rtruediv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.true_divide, '__rtruediv__', *args, **kwargs) def __floordiv__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.floor_divide, '__rfloordiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.floor_divide, '__rfloordiv__', *args, **kwargs) def __mod__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.mod, '__rdiv__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.mod, '__rdiv__', *args, **kwargs) def __pow__(self, other, modulo=None, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.power, '__rpower__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.power, '__rpower__', *args, **kwargs) def __lshift__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.left_shift, '__rlshift__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.left_shift, '__rlshift__', *args, **kwargs) def __rshift__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.right_shift, '__rrshift__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.right_shift, '__rrshift__', *args, **kwargs) def __and__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.bitwise_and, '__rand__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.bitwise_and, '__rand__', *args, **kwargs) def __or__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.bitwise_or, '__ror__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.bitwise_or, '__ror__', *args, **kwargs) def __xor__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.bitwise_xor, '__rxor__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.bitwise_xor, '__rxor__', *args, **kwargs) # Binary - right versions def __radd__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.add, '__add__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.add, '__add__', *args, **kwargs) def __rsub__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.subtract, '__sub__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.subtract, '__sub__', *args, **kwargs) def __rmul__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.multiply, '__mul__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.multiply, '__mul__', *args, **kwargs) def __rdiv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.divide, '__div__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.divide, '__div__', *args, **kwargs) def __rtruediv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.true_divide, '__truediv__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.true_divide, '__truediv__', *args, **kwargs) def __rfloordiv__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.floor_divide, '__floordiv__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.floor_divide, '__floordiv__', *args, **kwargs) def __rmod__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.mod, '__mod__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.mod, '__mod__', *args, **kwargs) def __rpow__(self, other, modulo=None, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.power, '__pow__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.power, '__pow__', *args, **kwargs) def __rlshift__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.left_shift, '__lshift__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.left_shift, '__lshift__', *args, **kwargs) def __rrshift__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.right_shift, '__rshift__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.right_shift, '__rshift__', *args, **kwargs) def __rand__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.bitwise_and, '__and__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.bitwise_and, '__and__', *args, **kwargs) def __ror__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.bitwise_or, '__or__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.bitwise_or, '__or__', *args, **kwargs) def __rxor__(self, other, *args, **kwargs): - return self._rbinary_op_from_ufunc(other, distarray.client.bitwise_xor, '__xor__', *args, **kwargs) + return self._rbinary_op_from_ufunc(other, distarray.dist.bitwise_xor, '__xor__', *args, **kwargs) def __neg__(self, *args, **kwargs): - return distarray.client.negative(self, *args, **kwargs) + return distarray.dist.negative(self, *args, **kwargs) def __pos__(self, *args, **kwargs): return self def __abs__(self, *args, **kwargs): - return distarray.client.abs(self, *args, **kwargs) + return distarray.dist.abs(self, *args, **kwargs) def __invert__(self, *args, **kwargs): - return distarray.client.invert(self, *args, **kwargs) + return distarray.dist.invert(self, *args, **kwargs) # Boolean comparisons def __lt__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.less, '__lt__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.less, '__lt__', *args, **kwargs) def __le__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.less_equal, '__le__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.less_equal, '__le__', *args, **kwargs) def __eq__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.equal, '__eq__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.equal, '__eq__', *args, **kwargs) def __ne__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.not_equal, '__ne__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.not_equal, '__ne__', *args, **kwargs) def __gt__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.greater, '__gt__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.greater, '__gt__', *args, **kwargs) def __ge__(self, other, *args, **kwargs): - return self._binary_op_from_ufunc(other, distarray.client.greater_equal, '__ge__', *args, **kwargs) + return self._binary_op_from_ufunc(other, distarray.dist.greater_equal, '__ge__', *args, **kwargs) diff --git a/distarray/client/functions.py b/distarray/dist/functions.py similarity index 98% rename from distarray/client/functions.py rename to distarray/dist/functions.py index 4b6b44d1..1e4d5138 100644 --- a/distarray/client/functions.py +++ b/distarray/dist/functions.py @@ -13,7 +13,7 @@ import numpy from distarray.error import ContextError -from distarray.client.distarray import DistArray +from distarray.dist.distarray import DistArray __all__ = [] # unary_names and binary_names added to __all__ below. diff --git a/distarray/client/maps.py b/distarray/dist/maps.py similarity index 100% rename from distarray/client/maps.py rename to distarray/dist/maps.py diff --git a/distarray/client/random.py b/distarray/dist/random.py similarity index 99% rename from distarray/client/random.py rename to distarray/dist/random.py index 5083d93b..35ac55e1 100644 --- a/distarray/client/random.py +++ b/distarray/dist/random.py @@ -9,8 +9,8 @@ from __future__ import absolute_import -from distarray.client.distarray import DistArray -from distarray.client.client_map import Distribution +from distarray.dist.distarray import DistArray +from distarray.dist.client_map import Distribution class Random(object): diff --git a/distarray/tests/test_client.py b/distarray/tests/test_client.py index eb83b993..5ae4e529 100644 --- a/distarray/tests/test_client.py +++ b/distarray/tests/test_client.py @@ -18,9 +18,9 @@ from numpy.testing import assert_array_equal, assert_allclose from distarray.externals.six.moves import range -from distarray.client.distarray import DistArray -from distarray.client.maps import Distribution -from distarray.client.context import Context +from distarray.dist.distarray import DistArray +from distarray.dist.maps import Distribution +from distarray.dist.context import Context class TestDistArray(unittest.TestCase): diff --git a/distarray/tests/test_client_map.py b/distarray/tests/test_client_map.py index 949ffc46..dd55b0da 100644 --- a/distarray/tests/test_client_map.py +++ b/distarray/tests/test_client_map.py @@ -9,8 +9,8 @@ from distarray.externals.six.moves import range -from distarray.client.context import Context -from distarray.client import maps as client_map +from distarray.dist.context import Context +from distarray.dist import maps as client_map class TestClientMap(unittest.TestCase): @@ -56,6 +56,3 @@ def test_2D_cc(self): rank = (r % nprocs_per_dim) * nprocs_per_dim + (c % nprocs_per_dim) actual = cm.owning_ranks((r,c)) self.assertSequenceEqual(actual, [rank]) - - - diff --git a/distarray/tests/test_context.py b/distarray/tests/test_context.py index 07b5dd01..f00ae5e0 100644 --- a/distarray/tests/test_context.py +++ b/distarray/tests/test_context.py @@ -17,7 +17,7 @@ import numpy -from distarray.client.context import Context +from distarray.dist.context import Context from distarray.ipython_utils import IPythonClient from distarray.local import LocalArray diff --git a/distarray/tests/test_decorators.py b/distarray/tests/test_decorators.py index 964b292a..a372deab 100644 --- a/distarray/tests/test_decorators.py +++ b/distarray/tests/test_decorators.py @@ -17,8 +17,8 @@ import numpy from numpy.testing import assert_array_equal -from distarray.client.context import Context -from distarray.client.decorators import DecoratorBase, local, vectorize +from distarray.dist.context import Context +from distarray.dist.decorators import DecoratorBase, local, vectorize from distarray.error import ContextError diff --git a/distarray/tests/test_distributed_io.py b/distarray/tests/test_distributed_io.py index 4c0c6dcd..e7e053a7 100644 --- a/distarray/tests/test_distributed_io.py +++ b/distarray/tests/test_distributed_io.py @@ -19,8 +19,8 @@ from distarray.externals.six.moves import range -from distarray.client.distarray import DistArray -from distarray.client.context import Context +from distarray.dist.distarray import DistArray +from distarray.dist.context import Context from distarray.testing import import_or_skip, temp_filepath diff --git a/distarray/tests/test_random.py b/distarray/tests/test_random.py index 9d610e1d..1a9a24b6 100644 --- a/distarray/tests/test_random.py +++ b/distarray/tests/test_random.py @@ -8,8 +8,8 @@ import unittest -from distarray.client.context import Context -from distarray.client.random import Random +from distarray.dist.context import Context +from distarray.dist.random import Random class TestRandom(unittest.TestCase): diff --git a/distarray/tests/test_umath.py b/distarray/tests/test_umath.py index 7b0922c8..69c4109c 100644 --- a/distarray/tests/test_umath.py +++ b/distarray/tests/test_umath.py @@ -16,8 +16,8 @@ import numpy as np from numpy.testing import assert_array_equal -import distarray.client.functions as functions -from distarray.client.context import Context +import distarray.dist.functions as functions +from distarray.dist.context import Context def add_checkers(cls, ops, checker_name): From a082a31525d67ba26f2901499fc9a165831ed8ff Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 29 Apr 2014 15:46:18 -0500 Subject: [PATCH 07/11] Remove cruft from bad rebase. --- distarray/dist/context.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/distarray/dist/context.py b/distarray/dist/context.py index be346f4d..f1840293 100644 --- a/distarray/dist/context.py +++ b/distarray/dist/context.py @@ -24,8 +24,6 @@ from distarray.ipython_utils import IPythonClient from distarray import DISTARRAY_BASE_NAME -atexit.register(cleanup.cleanup_all, DISTARRAY_BASE_NAME) - class Context(object): """ From 4455a0757fa38363db96d54626a3128ecd46a868 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 1 May 2014 19:38:26 -0500 Subject: [PATCH 08/11] Move client tests to dist/tests/. --- distarray/{ => dist}/tests/__init__.py | 0 distarray/{ => dist}/tests/test_client.py | 0 distarray/{ => dist}/tests/test_client_map.py | 0 distarray/{ => dist}/tests/test_context.py | 0 distarray/{ => dist}/tests/test_decorators.py | 0 distarray/{ => dist}/tests/test_distributed_io.py | 0 distarray/{ => dist}/tests/test_metadata_utils.py | 0 distarray/{ => dist}/tests/test_mpiutils.py | 0 distarray/{ => dist}/tests/test_random.py | 0 distarray/{ => dist}/tests/test_testing.py | 0 distarray/{ => dist}/tests/test_umath.py | 0 distarray/{ => dist}/tests/test_utils.py | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename distarray/{ => dist}/tests/__init__.py (100%) rename distarray/{ => dist}/tests/test_client.py (100%) rename distarray/{ => dist}/tests/test_client_map.py (100%) rename distarray/{ => dist}/tests/test_context.py (100%) rename distarray/{ => dist}/tests/test_decorators.py (100%) rename distarray/{ => dist}/tests/test_distributed_io.py (100%) rename distarray/{ => dist}/tests/test_metadata_utils.py (100%) rename distarray/{ => dist}/tests/test_mpiutils.py (100%) rename distarray/{ => dist}/tests/test_random.py (100%) rename distarray/{ => dist}/tests/test_testing.py (100%) rename distarray/{ => dist}/tests/test_umath.py (100%) rename distarray/{ => dist}/tests/test_utils.py (100%) diff --git a/distarray/tests/__init__.py b/distarray/dist/tests/__init__.py similarity index 100% rename from distarray/tests/__init__.py rename to distarray/dist/tests/__init__.py diff --git a/distarray/tests/test_client.py b/distarray/dist/tests/test_client.py similarity index 100% rename from distarray/tests/test_client.py rename to distarray/dist/tests/test_client.py diff --git a/distarray/tests/test_client_map.py b/distarray/dist/tests/test_client_map.py similarity index 100% rename from distarray/tests/test_client_map.py rename to distarray/dist/tests/test_client_map.py diff --git a/distarray/tests/test_context.py b/distarray/dist/tests/test_context.py similarity index 100% rename from distarray/tests/test_context.py rename to distarray/dist/tests/test_context.py diff --git a/distarray/tests/test_decorators.py b/distarray/dist/tests/test_decorators.py similarity index 100% rename from distarray/tests/test_decorators.py rename to distarray/dist/tests/test_decorators.py diff --git a/distarray/tests/test_distributed_io.py b/distarray/dist/tests/test_distributed_io.py similarity index 100% rename from distarray/tests/test_distributed_io.py rename to distarray/dist/tests/test_distributed_io.py diff --git a/distarray/tests/test_metadata_utils.py b/distarray/dist/tests/test_metadata_utils.py similarity index 100% rename from distarray/tests/test_metadata_utils.py rename to distarray/dist/tests/test_metadata_utils.py diff --git a/distarray/tests/test_mpiutils.py b/distarray/dist/tests/test_mpiutils.py similarity index 100% rename from distarray/tests/test_mpiutils.py rename to distarray/dist/tests/test_mpiutils.py diff --git a/distarray/tests/test_random.py b/distarray/dist/tests/test_random.py similarity index 100% rename from distarray/tests/test_random.py rename to distarray/dist/tests/test_random.py diff --git a/distarray/tests/test_testing.py b/distarray/dist/tests/test_testing.py similarity index 100% rename from distarray/tests/test_testing.py rename to distarray/dist/tests/test_testing.py diff --git a/distarray/tests/test_umath.py b/distarray/dist/tests/test_umath.py similarity index 100% rename from distarray/tests/test_umath.py rename to distarray/dist/tests/test_umath.py diff --git a/distarray/tests/test_utils.py b/distarray/dist/tests/test_utils.py similarity index 100% rename from distarray/tests/test_utils.py rename to distarray/dist/tests/test_utils.py From dcd859b5833ee86a23d185a180eb7c7004ad84e5 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2014 11:52:50 -0500 Subject: [PATCH 09/11] Fix a missed instance of client_map -> maps --- distarray/dist/random.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distarray/dist/random.py b/distarray/dist/random.py index 35ac55e1..245e213c 100644 --- a/distarray/dist/random.py +++ b/distarray/dist/random.py @@ -10,7 +10,7 @@ from __future__ import absolute_import from distarray.dist.distarray import DistArray -from distarray.dist.client_map import Distribution +from distarray.dist.maps import Distribution class Random(object): From 4c064e6a7232cdbaeb009e007806d644fd662360 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2014 13:16:23 -0500 Subject: [PATCH 10/11] Moves suggested by @kwmsmith --- distarray/{ => dist}/cleanup.py | 6 +++--- distarray/dist/context.py | 12 ++++++------ distarray/{ => dist}/ipython_utils.py | 0 distarray/dist/tests/test_context.py | 2 +- distarray/local/construct.py | 2 +- distarray/local/localarray.py | 2 +- distarray/{ => local}/mpiutils.py | 0 distarray/local/tests/paralleltest_mpiutils.py | 3 ++- distarray/testing.py | 2 +- 9 files changed, 15 insertions(+), 14 deletions(-) rename distarray/{ => dist}/cleanup.py (95%) rename distarray/{ => dist}/ipython_utils.py (100%) rename distarray/{ => local}/mpiutils.py (100%) diff --git a/distarray/cleanup.py b/distarray/dist/cleanup.py similarity index 95% rename from distarray/cleanup.py rename to distarray/dist/cleanup.py index 06d444ab..8e967c69 100644 --- a/distarray/cleanup.py +++ b/distarray/dist/cleanup.py @@ -4,9 +4,9 @@ # Distributed under the terms of the BSD License. See COPYING.rst. # --------------------------------------------------------------------------- -from __future__ import print_function +from __future__ import print_function, absolute_import -from distarray.ipython_utils import IPythonClient +from distarray.dist.ipython_utils import IPythonClient def cleanup(view, module_name, prefix): """ Delete Context object with the given name from the given module""" @@ -46,7 +46,7 @@ def get_keys_engine(prefix): for target, keys in keys_from_target.items(): for key in keys: targets_from_key.setdefault(key, []).append(target) - + return targets_from_key def clear(view): diff --git a/distarray/dist/context.py b/distarray/dist/context.py index f1840293..7f3cc7c9 100644 --- a/distarray/dist/context.py +++ b/distarray/dist/context.py @@ -16,12 +16,12 @@ import numpy -from distarray import cleanup +from distarray.dist import cleanup from distarray.externals import six from distarray.dist.distarray import DistArray from distarray.dist.maps import Distribution -from distarray.ipython_utils import IPythonClient +from distarray.dist.ipython_utils import IPythonClient from distarray import DISTARRAY_BASE_NAME @@ -70,7 +70,7 @@ def __init__(self, client=None, targets=None): #with self.view.sync_imports(): # import distarray self.view.execute("import distarray.local; " - "import distarray.mpiutils; " + "import distarray.local.mpiutils; " "import numpy") self.context_key = self._setup_context_key() @@ -91,13 +91,13 @@ def _setup_context_key(self): def _make_intracomm(self): def get_rank(): - from distarray.mpiutils import COMM_PRIVATE + from distarray.local.mpiutils import COMM_PRIVATE return COMM_PRIVATE.Get_rank() # self.view's engines must encompass all ranks in the MPI communicator, # i.e., everything in rank_map.values(). def get_size(): - from distarray.mpiutils import COMM_PRIVATE + from distarray.local.mpiutils import COMM_PRIVATE return COMM_PRIVATE.Get_size() # get a mapping of IPython engine ID to MPI rank @@ -112,7 +112,7 @@ def get_size(): # MPI_Comm_create must be called on all engines, not just those # involved in the new communicator. comm_key = self._generate_key() - cmd = "%s = distarray.mpiutils.create_comm_with_list(%s)" + cmd = "%s = distarray.local.mpiutils.create_comm_with_list(%s)" cmd %= (comm_key, ranks) self.view.execute(cmd, block=True) return comm_key diff --git a/distarray/ipython_utils.py b/distarray/dist/ipython_utils.py similarity index 100% rename from distarray/ipython_utils.py rename to distarray/dist/ipython_utils.py diff --git a/distarray/dist/tests/test_context.py b/distarray/dist/tests/test_context.py index f00ae5e0..0fb3f72e 100644 --- a/distarray/dist/tests/test_context.py +++ b/distarray/dist/tests/test_context.py @@ -18,7 +18,7 @@ import numpy from distarray.dist.context import Context -from distarray.ipython_utils import IPythonClient +from distarray.dist.ipython_utils import IPythonClient from distarray.local import LocalArray diff --git a/distarray/local/construct.py b/distarray/local/construct.py index 994fcd8c..7a285358 100644 --- a/distarray/local/construct.py +++ b/distarray/local/construct.py @@ -6,7 +6,7 @@ from __future__ import division -from distarray.mpiutils import MPI +from distarray.local.mpiutils import MPI from distarray.local.error import NullCommError, InvalidBaseCommError from distarray import mpiutils diff --git a/distarray/local/localarray.py b/distarray/local/localarray.py index 9386061c..b15f558d 100644 --- a/distarray/local/localarray.py +++ b/distarray/local/localarray.py @@ -19,7 +19,7 @@ from distarray.externals import six from distarray.externals.six.moves import zip -from distarray.mpiutils import MPI +from distarray.local.mpiutils import MPI from distarray.utils import _raise_nie from distarray.local import format, maps from distarray.local.error import InvalidDimensionError, IncompatibleArrayError diff --git a/distarray/mpiutils.py b/distarray/local/mpiutils.py similarity index 100% rename from distarray/mpiutils.py rename to distarray/local/mpiutils.py diff --git a/distarray/local/tests/paralleltest_mpiutils.py b/distarray/local/tests/paralleltest_mpiutils.py index 89e1e407..39483c41 100644 --- a/distarray/local/tests/paralleltest_mpiutils.py +++ b/distarray/local/tests/paralleltest_mpiutils.py @@ -11,7 +11,8 @@ from distarray.local.localarray import zeros from distarray.local.maps import Distribution from distarray.error import InvalidCommSizeError, InvalidRankError -from distarray.mpiutils import MPI, create_comm_of_size, create_comm_with_list +from distarray.local.mpiutils import (MPI, create_comm_of_size, + create_comm_with_list) class TestCreateCommAlternate(unittest.TestCase): diff --git a/distarray/testing.py b/distarray/testing.py index b66a1cab..d806cf4b 100644 --- a/distarray/testing.py +++ b/distarray/testing.py @@ -20,7 +20,7 @@ from distarray.externals import protocol_validator from distarray.error import InvalidCommSizeError -from distarray.mpiutils import MPI, create_comm_of_size +from distarray.local.mpiutils import MPI, create_comm_of_size def raise_typeerror(fn): From 767e989157be414431f5fd5948741ec832e76772 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 2 May 2014 16:02:09 -0500 Subject: [PATCH 11/11] More moves and rebase fixes --- distarray/apps/dacluster.py | 2 +- distarray/dist/tests/{test_client.py => test_distarray.py} | 0 distarray/dist/tests/{test_umath.py => test_functions.py} | 0 distarray/dist/tests/{test_client_map.py => test_maps.py} | 0 distarray/local/construct.py | 2 +- distarray/{dist => local}/tests/test_mpiutils.py | 2 +- distarray/{dist => }/tests/test_metadata_utils.py | 0 distarray/{dist => }/tests/test_testing.py | 0 distarray/{dist => }/tests/test_utils.py | 0 9 files changed, 3 insertions(+), 3 deletions(-) rename distarray/dist/tests/{test_client.py => test_distarray.py} (100%) rename distarray/dist/tests/{test_umath.py => test_functions.py} (100%) rename distarray/dist/tests/{test_client_map.py => test_maps.py} (100%) rename distarray/{dist => local}/tests/test_mpiutils.py (93%) rename distarray/{dist => }/tests/test_metadata_utils.py (100%) rename distarray/{dist => }/tests/test_testing.py (100%) rename distarray/{dist => }/tests/test_utils.py (100%) diff --git a/distarray/apps/dacluster.py b/distarray/apps/dacluster.py index f5e08dc5..a44b4e54 100755 --- a/distarray/apps/dacluster.py +++ b/distarray/apps/dacluster.py @@ -18,7 +18,7 @@ from subprocess import Popen, PIPE from distarray.externals import six -from distarray.cleanup import clear_all +from distarray.dist.cleanup import clear_all if six.PY2: diff --git a/distarray/dist/tests/test_client.py b/distarray/dist/tests/test_distarray.py similarity index 100% rename from distarray/dist/tests/test_client.py rename to distarray/dist/tests/test_distarray.py diff --git a/distarray/dist/tests/test_umath.py b/distarray/dist/tests/test_functions.py similarity index 100% rename from distarray/dist/tests/test_umath.py rename to distarray/dist/tests/test_functions.py diff --git a/distarray/dist/tests/test_client_map.py b/distarray/dist/tests/test_maps.py similarity index 100% rename from distarray/dist/tests/test_client_map.py rename to distarray/dist/tests/test_maps.py diff --git a/distarray/local/construct.py b/distarray/local/construct.py index 7a285358..5908fca5 100644 --- a/distarray/local/construct.py +++ b/distarray/local/construct.py @@ -8,7 +8,7 @@ from distarray.local.mpiutils import MPI from distarray.local.error import NullCommError, InvalidBaseCommError -from distarray import mpiutils +from distarray.local import mpiutils # --------------------------------------------------------------------------- diff --git a/distarray/dist/tests/test_mpiutils.py b/distarray/local/tests/test_mpiutils.py similarity index 93% rename from distarray/dist/tests/test_mpiutils.py rename to distarray/local/tests/test_mpiutils.py index 160f8fca..11512c3a 100644 --- a/distarray/dist/tests/test_mpiutils.py +++ b/distarray/local/tests/test_mpiutils.py @@ -7,7 +7,7 @@ import unittest import numpy -from distarray.mpiutils import mpi_type_for_ndarray +from distarray.local.mpiutils import mpi_type_for_ndarray class TestMpiTypes(unittest.TestCase): diff --git a/distarray/dist/tests/test_metadata_utils.py b/distarray/tests/test_metadata_utils.py similarity index 100% rename from distarray/dist/tests/test_metadata_utils.py rename to distarray/tests/test_metadata_utils.py diff --git a/distarray/dist/tests/test_testing.py b/distarray/tests/test_testing.py similarity index 100% rename from distarray/dist/tests/test_testing.py rename to distarray/tests/test_testing.py diff --git a/distarray/dist/tests/test_utils.py b/distarray/tests/test_utils.py similarity index 100% rename from distarray/dist/tests/test_utils.py rename to distarray/tests/test_utils.py