From 402a86f22211d634a09c9cc9bc477ae4a855debf Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 22 Apr 2014 12:19:39 -0500 Subject: [PATCH 1/2] Remove a bunch of Not implemented functions. And unused headings. Also removes the code section numbering since it is no longer accurate. I'll add a section to the wiki of all the NIE localarray functions which were removed so that we can implement them later. --- distarray/local/localarray.py | 431 +--------------------------------- 1 file changed, 10 insertions(+), 421 deletions(-) diff --git a/distarray/local/localarray.py b/distarray/local/localarray.py index ad15bf3a..d2f4c6e1 100644 --- a/distarray/local/localarray.py +++ b/distarray/local/localarray.py @@ -523,9 +523,9 @@ def global_limits(self, dim): return lower_global[dim], upper_global[dim] #------------------------------------------------------------------------- - # 3.2 ndarray methods + # ndarray methods #------------------------------------------------------------------------- - # 3.2.1 Array conversion + # Array conversion #------------------------------------------------------------------------- def astype(self, newdtype): @@ -600,81 +600,6 @@ def __array_wrap__(self, obj, context=None): def fill(self, scalar): self.local_array.fill(scalar) - #------------------------------------------------------------------------- - # 3.2.2 Array shape manipulation - #------------------------------------------------------------------------- - - def reshape(self, newshape): - _raise_nie() - - def redist(self, newshape, newdist={0: 'b'}, newgrid_shape=None): - _raise_nie() - - def resize(self, newshape, refcheck=1, order='C'): - _raise_nie() - - def transpose(self, axes=None): - _raise_nie() - - def swapaxes(self, axis1, axis2): - _raise_nie() - - def flatten(self, order='C'): - _raise_nie() - - def ravel(self, order='C'): - _raise_nie() - - def squeeze(self): - _raise_nie() - - def asdist(self, shape, dist={0: 'b'}, grid_shape=None): - pass - # new_da = LocalArray(shape, self.dtype, dist, grid_shape, - # self.base_comm) - # base_comm = self.base_comm - # local_array = self.local_array - # new_local_array = da.local_array - # recv_counts = np.zeros(self.comm_size, dtype=int) - # - # status = MPI.Status() - # MPI.Attach_buffer(np.empty(128+MPI.BSEND_OVERHEAD,dtype=float)) - # done_count = 0 - # - # for old_local_inds, item in np.ndenumerate(local_array): - # - # # Compute the new owner - # global_inds = self.global_from_local(new_da.comm_rank, - # old_local_inds) - # new_owner = new_da.owner_rank(global_inds) - # if new_owner==self.owner_rank: - # pass - # # Just move the data to the right place in new_local_array - # else: - # # Send to the new owner with default tag - # # Bsend is probably best, but Isend is also a possibility. - # request = comm.Isend(item, dest=new_owner) - # - # # Recv - # incoming = comm.Iprobe(MPI.ANY_SOURCE, MPI.ANY_TAG, status) - # if incoming: - # old_owner = status.Get_source() - # tag = status.Get_tag() - # data = comm.Recv(old_owner, tag) - # if tag==2: - # done_count += 1 - # # Figure out where new location of old_owner, tag - # new_local_ind = local_ind_by_owner_and_location(old_owner, - # location) - # new_local_array[new_local_ind] = y - # recv_counts[old_owner] = recv_counts[old_owner]+1 - # - # while done_count < self.comm_size: - # pass - # - # - # MPI.Detach_buffer() - def asdist_like(self, other): """ Return a version of self that has shape, dist and grid_shape like @@ -686,76 +611,6 @@ def asdist_like(self, other): raise IncompatibleArrayError("DistArrays have incompatible shape," "dist or grid_shape") - #------------------------------------------------------------------------- - # 3.2.3 Array item selection and manipulation - #------------------------------------------------------------------------- - - def take(self, indices, axis=None, out=None, mode='raise'): - _raise_nie() - - def put(self, indices, values, mode='raise'): - _raise_nie() - - def putmask(self, mask, values): - _raise_nie() - - def repeat(self, repeats, axis=None): - _raise_nie() - - def choose(self, choices, out=None, mode='raise'): - _raise_nie() - - def sort(self, axis=-1, kind='quick'): - _raise_nie() - - def argsort(self, axis=-1, kind='quick'): - _raise_nie() - - def searchsorted(self, values): - _raise_nie() - - def nonzero(self): - _raise_nie() - - def compress(self, condition, axis=None, out=None): - _raise_nie() - - def diagonal(self, offset=0, axis1=0, axis2=1): - _raise_nie() - - #------------------------------------------------------------------------- - # 3.2.4 Array item selection and manipulation - #------------------------------------------------------------------------- - - def max(self, axis=None, out=None): - _raise_nie() - - def argmax(self, axis=None, out=None): - _raise_nie() - - def min(axis=None, out=None): - _raise_nie() - - def argmin(self, axis=None, out=None): - _raise_nie() - - def ptp(self, axis=None, out=None): - _raise_nie() - - def clip(self, min, max, out=None): - _raise_nie() - - def conj(self, out=None): - _raise_nie() - - conjugate = conj - - def round(self, decimals=0, out=None): - _raise_nie() - - def trace(self, offset=0, axis1=0, axis2=1, dtype=None, out=None): - _raise_nie() - #TODO FIXME: implement axis and out kwargs. def sum(self, axis=None, dtype=None, out=None): if axis or out is not None: @@ -788,37 +643,8 @@ def std(self, axis=None, dtype=None, out=None): else: return math.sqrt(self.var()) - def cumsum(self, axis=None, dtype=None, out=None): - _raise_nie() - - def prod(self, axis=None, dtype=None, out=None): - _raise_nie() - - def cumprod(self, axis=None, dtype=None, out=None): - _raise_nie() - - def all(self, axis=None, out=None): - _raise_nie() - - def any(self, axis=None, out=None): - _raise_nie() - - #------------------------------------------------------------------------- - # 3.3 Array special methods - #------------------------------------------------------------------------- - #------------------------------------------------------------------------- - # 3.3.1 Methods for standard library functions - #------------------------------------------------------------------------- - - def __copy__(self): - _raise_nie() - - def __deepcopy__(self): - _raise_nie() - - #------------------------------------------------------------------------- - # 3.3.2 Basic customization + # Basic customization #------------------------------------------------------------------------- def __lt__(self, other): @@ -845,11 +671,8 @@ def __str__(self): def __repr__(self): return str(self.local_array) - def __nonzero__(self): - _raise_nie() - #------------------------------------------------------------------------- - # 3.3.3 Container customization + # Container customization #------------------------------------------------------------------------- def __len__(self): @@ -895,7 +718,7 @@ def unpack_index(self, packed_ind): return tuple(packed_ind//strides_array % self.global_shape) #-------------------------------------------------------------------------- - # 3.3.4 Arithmetic customization - binary + # Arithmetic customization - binary #-------------------------------------------------------------------------- # Binary @@ -935,9 +758,6 @@ def __floordiv__(self, other): def __mod__(self, other): return self._binary_op_from_ufunc(other, mod, '__rdiv__') - def __divmod__(self, other): - _raise_nie() - def __pow__(self, other, modulo=None): return self._binary_op_from_ufunc(other, power, '__rpower__') @@ -979,9 +799,6 @@ def __rfloordiv__(self, other): def __rmod__(self, other): return self._rbinary_op_from_ufunc(other, mod, '__mod__') - def __rdivmod__(self, other): - _raise_nie() - def __rpow__(self, other, modulo=None): return self._rbinary_op_from_ufunc(other, power, '__pow__') @@ -1000,47 +817,6 @@ def __ror__(self, other): def __rxor__(self, other): return self._rbinary_op_from_ufunc(other, bitwise_xor, '__xor__') - # Inplace - - def __iadd__(self, other): - _raise_nie() - - def __isub__(self, other): - _raise_nie() - - def __imul__(self, other): - _raise_nie() - - def __idiv__(self, other): - _raise_nie() - - def __itruediv__(self, other): - _raise_nie() - - def __ifloordiv__(self, other): - _raise_nie() - - def __imod__(self, other): - _raise_nie() - - def __ipow__(self, other, modulo=None): - _raise_nie() - - def __ilshift__(self, other): - _raise_nie() - - def __irshift__(self, other): - _raise_nie() - - def __iand__(self, other): - _raise_nie() - - def __ior__(self, other): - _raise_nie() - - def __ixor__(self, other): - _raise_nie() - # Unary def __neg__(self): @@ -1073,26 +849,9 @@ def __invert__(self): # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- -# Utilities needed to implement things below -# --------------------------------------------------------------------------- - -# --------------------------------------------------------------------------- -# 4 Basic routines +# Creating arrays # --------------------------------------------------------------------------- -# --------------------------------------------------------------------------- -# 4.1 Creating arrays -# --------------------------------------------------------------------------- - -def aslocalarray(object, dtype=None, order=None): - _raise_nie() - - -def arange(start, stop=None, step=1, dtype=None, dist={0: 'b'}, - grid_shape=None, comm=None): - _raise_nie() - - def empty(shape, dtype=float, dist=None, grid_shape=None, comm=None): return LocalArray(shape, dtype=dtype, dist=dist, grid_shape=grid_shape, comm=comm) @@ -1393,17 +1152,8 @@ def fromlocalarray_like(local_arr, like_arr): like_arr.grid_shape, like_arr.base_comm, buf=local_arr) return res - -def identity(n, dtype=np.intp): - _raise_nie() - - -def where(condition, x=None, y=None): - _raise_nie() - - # --------------------------------------------------------------------------- -# 4.2 Operations on two or more arrays +# Operations on two or more arrays # --------------------------------------------------------------------------- def arecompatible(a, b): @@ -1411,55 +1161,6 @@ def arecompatible(a, b): return a.compatibility_hash() == b.compatibility_hash() -def concatenate(seq, axis=0): - _raise_nie() - - -def correlate(x, y, mode='valid'): - _raise_nie() - - -def convolve(x, y, mode='valid'): - _raise_nie() - - -def outer(a, b): - _raise_nie() - - -def inner(a, b): - _raise_nie() - - -def dot(a, b): - _raise_nie() - - -def vdot(a, b): - _raise_nie() - - -def tensordot(a, b, axes=(-1, 0)): - _raise_nie() - - -def cross(a, b, axisa=-1, axisb=-1, axisc=-1, axis=None): - _raise_nie() - - -def allclose(a, b, rtol=10e-5, atom=10e-8): - _raise_nie() - - -# --------------------------------------------------------------------------- -# 4.3 Printing arrays -# --------------------------------------------------------------------------- - - -def distarray2string(a): - _raise_nie() - - def set_printoptions(precision=None, threshold=None, edgeitems=None, linewidth=None, suppress=None): res = np.set_printoptions(precision, threshold, edgeitems, linewidth, @@ -1472,7 +1173,7 @@ def get_printoptions(): # --------------------------------------------------------------------------- -# 4.5 Dealing with data types +# Dealing with data types # --------------------------------------------------------------------------- @@ -1485,17 +1186,7 @@ def get_printoptions(): # --------------------------------------------------------------------------- -# 5 Additional convenience routines -# --------------------------------------------------------------------------- - - -# --------------------------------------------------------------------------- -# 5.1 Shape functions -# --------------------------------------------------------------------------- - - -# --------------------------------------------------------------------------- -# 5.2 Basic functions +# Basic functions # --------------------------------------------------------------------------- @@ -1505,77 +1196,8 @@ def sum(a, dtype=None): return global_sum -def average(a, axis=None, weights=None, returned=0): - _raise_nie() - - -def cov(x, y=None, rowvar=1, bias=0): - _raise_nie() - - -def corrcoef(x, y=None, rowvar=1, bias=0): - _raise_nie() - - -def median(m): - _raise_nie() - - -def digitize(x, bins): - _raise_nie() - - -def histogram(x, bins=None, range=None, normed=False): - _raise_nie() - - -def histogram2d(x, y, bins, normed=False): - _raise_nie() - - -def logspace(start, stop, num=50, endpoint=True, base=10.0): - _raise_nie() - - -def linspace(start, stop, num=50, endpoint=True, retstep=False): - _raise_nie() - - -# --------------------------------------------------------------------------- -# 5.3 Polynomial functions -# --------------------------------------------------------------------------- - - -# --------------------------------------------------------------------------- -# 5.4 Set operations -# --------------------------------------------------------------------------- - - # --------------------------------------------------------------------------- -# 5.5 Array construction using index tricks -# --------------------------------------------------------------------------- - - -# --------------------------------------------------------------------------- -# 5.6 Other indexing devices -# --------------------------------------------------------------------------- - - -# --------------------------------------------------------------------------- -# 5.7 Two-dimensional functions -# --------------------------------------------------------------------------- - - -def eye(n, m=None, k=0, dtype=float): - _raise_nie() - - -def diag(v, k=0): - _raise_nie() - - -# --------------------------------------------------------------------------- -# 5.8 More data type functions +# More data type functions # --------------------------------------------------------------------------- @@ -1590,39 +1212,6 @@ def diag(v, k=0): mintypecode = np.mintypecode finfo = np.finfo - -# --------------------------------------------------------------------------- -# 5.9 Functions that behave like ufuncs -# --------------------------------------------------------------------------- - - -# --------------------------------------------------------------------------- -# 5.10 Misc functions -# --------------------------------------------------------------------------- - - -# --------------------------------------------------------------------------- -# 5.11 Utility functions -# --------------------------------------------------------------------------- - - -# --------------------------------------------------------------------------- -# --------------------------------------------------------------------------- -# Universal Functions -# -# I would really like these functions to be in a separate file, but that -# is not possible because of circular import problems. Basically, these -# functions need access to the LocalArray object in this module, and the -# LocalArray object needs to use these functions. There are 3 options for -# solving this problem: -# -# * Put everything in one file -# * Put the functions needed by LocalArray in distarray, others elsewhere -# * Make a subclass of LocalArray that has methods that use the functions -# --------------------------------------------------------------------------- -# --------------------------------------------------------------------------- - - # Functions for manipulating shapes according to the broadcast rules. def _expand_shape(s, length, element=1): From 38e5b06d9d472eedaa25fec815b1098eed7ae408 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 22 Apr 2014 14:39:49 -0500 Subject: [PATCH 2/2] Remove tests for notimplemented methods. --- .../local/tests/paralleltest_localarray.py | 91 ------------------- 1 file changed, 91 deletions(-) diff --git a/distarray/local/tests/paralleltest_localarray.py b/distarray/local/tests/paralleltest_localarray.py index 1634f34f..8aba4817 100644 --- a/distarray/local/tests/paralleltest_localarray.py +++ b/distarray/local/tests/paralleltest_localarray.py @@ -621,97 +621,6 @@ def test_asdist_like(self): self.assertRaises(IncompatibleArrayError, a.asdist_like, b) -class TestNotImplementedArrayMethods(MpiTestCase): - """ Test that the not implemented functions can be called, - and raise an exception. As these methods get implemented, - they will start failing this test, and can be removed from it. - Eventually this test case should become empty! """ - - def setUp(self): - # These would need real values for a real test, - # but since we just check for not implemented, - # this is not necessary here. - self.larr1 = LocalArray((4,), comm=self.comm) - self.larr2 = LocalArray((4, 4), comm=self.comm) - self.larrb = LocalArray((4, 4), dtype=bool, comm=self.comm) - self.larrc = LocalArray((4, 4), dtype=complex, comm=self.comm) - - def test_array_shape_manipulation(self): - """ Array shape manipulation functions. """ - with self.assertRaises(NotImplementedError): - self.larr2.reshape((8, 2)) - with self.assertRaises(NotImplementedError): - self.larr2.redist((8, 2), newdist={0: 'n'}) - with self.assertRaises(NotImplementedError): - self.larr2.resize((8, 4)) - with self.assertRaises(NotImplementedError): - self.larr2.transpose(None) - with self.assertRaises(NotImplementedError): - self.larr2.swapaxes(0, 1) - with self.assertRaises(NotImplementedError): - self.larr2.flatten() - with self.assertRaises(NotImplementedError): - self.larr2.ravel() - with self.assertRaises(NotImplementedError): - self.larr2.squeeze() - - def test_array_item_selection_323(self): - """ Array item selection functions 3.2.3. """ - with self.assertRaises(NotImplementedError): - self.larr2.take([[0, 0], [1, 1]]) - with self.assertRaises(NotImplementedError): - self.larr2.put([0, 3], [42, 27]) - with self.assertRaises(NotImplementedError): - self.larr1.putmask([True, True, False, False], [12, 47]) - with self.assertRaises(NotImplementedError): - self.larr1.repeat(3) - with self.assertRaises(NotImplementedError): - self.larr1.choose([[1], [2]]) - with self.assertRaises(NotImplementedError): - self.larr1.sort() - with self.assertRaises(NotImplementedError): - self.larr1.argsort() - with self.assertRaises(NotImplementedError): - self.larr1.searchsorted([2, 3, 4]) - with self.assertRaises(NotImplementedError): - self.larr1.nonzero() - with self.assertRaises(NotImplementedError): - self.larr2.compress([False, True, False, True]) - with self.assertRaises(NotImplementedError): - self.larr2.diagonal() - - def test_array_item_selection_324(self): - """ Array item selection functions 3.2.4. """ - with self.assertRaises(NotImplementedError): - self.larr1.max() - with self.assertRaises(NotImplementedError): - self.larr1.argmax() - with self.assertRaises(NotImplementedError): - self.larr1.min() - with self.assertRaises(NotImplementedError): - self.larr1.argmin() - with self.assertRaises(NotImplementedError): - self.larr1.ptp() - with self.assertRaises(NotImplementedError): - self.larr1.clip(2, 5) - with self.assertRaises(NotImplementedError): - self.larrc.conj() - with self.assertRaises(NotImplementedError): - self.larr2.round() - with self.assertRaises(NotImplementedError): - self.larr2.trace() - with self.assertRaises(NotImplementedError): - self.larr2.cumsum() - with self.assertRaises(NotImplementedError): - self.larr2.prod() - with self.assertRaises(NotImplementedError): - self.larr2.cumprod() - with self.assertRaises(NotImplementedError): - self.larrb.all() - with self.assertRaises(NotImplementedError): - self.larrb.any() - - if __name__ == '__main__': try: unittest.main()