From 53ecae68f6504e103b00d31cd5686c86e239912d Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 1 May 2014 12:15:20 -0500 Subject: [PATCH 1/5] New __get/setitem__ with one roundtrip. --- distarray/dist/distarray.py | 49 +++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/distarray/dist/distarray.py b/distarray/dist/distarray.py index 8fd0853e..abfa3608 100644 --- a/distarray/dist/distarray.py +++ b/distarray/dist/distarray.py @@ -158,21 +158,33 @@ def __getitem__(self, index): # especially for special cases like `index == slice(None)`. # This would dramatically improve tondarray's performance. + def getit(index, self_key): + main_module = __import__('__main__') + # LOL the next line is bad. + arr = reduce(getattr, [main_module] + self_key.split('.')) + res = arr.checked_getitem(index) + return res + if isinstance(index, int) or isinstance(index, slice): tuple_index = (index,) return self.__getitem__(tuple_index) elif isinstance(index, tuple): targets = self.distribution.owning_targets(index) - result_key = self.context._generate_key() - fmt = '%s = %s.checked_getitem(%s)' - statement = fmt % (result_key, self.key, index) - self.context._execute(statement, targets=targets) - result = process_return_value(self.context, result_key, targets=targets) - if result is None: + + # Using a private IPython function is probably a bad think to do. + # but its the only apply function that lets us specify `targets`. + args = (index, self.key) + result = self.context.view._really_apply(getit, args=args, + targets=targets, block=True) + result = [i for i in result if i is not None] + if len(result) != 1: + raise IndexError("Getting more than one result (%s) is not " + " supported yet." % (result,)) + elif result is None: raise IndexError("Index %r is out of bounds" % (index,)) else: - return result + return result[0] else: raise TypeError("Invalid index type.") @@ -183,20 +195,27 @@ def __setitem__(self, index, value): # `value` and assign to local arrays. This would dramatically # improve the fromndarray method's performance. + def setit(index, self_key, value): + main_module = __import__('__main__') + arr = reduce(getattr, [main_module] + self_key.split('.')) + res = arr.checked_setitem(index, value) + return res + if isinstance(index, int) or isinstance(index, slice): tuple_index = (index,) return self.__setitem__(tuple_index, value) elif isinstance(index, tuple): targets = self.distribution.owning_targets(index) - result_key = self.context._generate_key() - fmt = '%s = %s.checked_setitem(%s, %s)' - statement = fmt % (result_key, self.key, index, value) - self.context._execute(statement, targets=targets) - result = process_return_value(self.context, result_key, targets=targets) - if result is None: - raise IndexError("Index %r is out of bounds" % (index,)) - + args = (index, self.key, value) + result = self.context.view._really_apply(setit, args=args, + targets=targets, block=True) + result = [i for i in result if i is not None] + if len(result) > 1: + raise IndexError("Setting more than one result (%s) is not " + " supported yet." % (result,)) + elif result == []: + raise IndexError("Index %s is out of bounds" % (index,)) else: raise TypeError("Invalid index type.") From f92066e0f04aaa0540cb160ec5e04a05620591b8 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 1 May 2014 13:24:34 -0500 Subject: [PATCH 2/5] Python 3 compatibility. Import functools.reduce. --- distarray/dist/distarray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/distarray/dist/distarray.py b/distarray/dist/distarray.py index abfa3608..fb7349bb 100644 --- a/distarray/dist/distarray.py +++ b/distarray/dist/distarray.py @@ -16,6 +16,7 @@ import operator from itertools import product +from functools import reduce import numpy as np From 1eaf7bf291fe1d206de19e7a68a60495ffe79233 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 1 May 2014 13:35:55 -0500 Subject: [PATCH 3/5] Remove unused function. --- distarray/dist/distarray.py | 38 ------------------------------------- 1 file changed, 38 deletions(-) diff --git a/distarray/dist/distarray.py b/distarray/dist/distarray.py index fb7349bb..c2e0ee6b 100644 --- a/distarray/dist/distarray.py +++ b/distarray/dist/distarray.py @@ -32,44 +32,6 @@ # Code # --------------------------------------------------------------------------- -def process_return_value(subcontext, result_key, targets): - """Figure out what to return on the Client. - - Parameters - ---------- - key : string - Key corresponding to wrapped function's return value. - - Returns - ------- - A DistArray (if locally all values are DistArray), a None (if - locally all values are None), or else, pull the result back to the - client and return it. If all but one of the pulled values is None, - return that non-None value only. - """ - type_key = subcontext._generate_key() - type_statement = "{} = str(type({}))".format(type_key, result_key) - subcontext._execute(type_statement, targets=targets) - result_type_str = subcontext._pull(type_key, targets=targets) - - def is_NoneType(typestring): - return (typestring == "" or - typestring == "") - - def is_LocalArray(typestring): - return typestring == "" - - if all(is_LocalArray(r) for r in result_type_str): - result = DistArray.from_localarrays(result_key, context=subcontext) - elif all(is_NoneType(r) for r in result_type_str): - result = None - else: - result = subcontext._pull(result_key, targets=targets) - if has_exactly_one(result): - result = next(x for x in result if x is not None) - - return result - _DIM_DATA_PER_RANK = """ {ddpr_name} = {local_name}.dim_data """ From 343023132854554d323646f46624f71810438c97 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Fri, 9 May 2014 11:48:13 -0500 Subject: [PATCH 4/5] Use new context.apply function in __get/setitem__. --- distarray/dist/distarray.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/distarray/dist/distarray.py b/distarray/dist/distarray.py index c2e0ee6b..de731dc3 100644 --- a/distarray/dist/distarray.py +++ b/distarray/dist/distarray.py @@ -121,12 +121,9 @@ def __getitem__(self, index): # especially for special cases like `index == slice(None)`. # This would dramatically improve tondarray's performance. - def getit(index, self_key): - main_module = __import__('__main__') - # LOL the next line is bad. - arr = reduce(getattr, [main_module] + self_key.split('.')) - res = arr.checked_getitem(index) - return res + # func that runs locally + def getit(arr, index): + return arr.checked_getitem(index) if isinstance(index, int) or isinstance(index, slice): tuple_index = (index,) @@ -137,9 +134,9 @@ def getit(index, self_key): # Using a private IPython function is probably a bad think to do. # but its the only apply function that lets us specify `targets`. - args = (index, self.key) - result = self.context.view._really_apply(getit, args=args, - targets=targets, block=True) + args = (self.key, index) + result = self.context.apply(getit, args=args, + targets=targets) result = [i for i in result if i is not None] if len(result) != 1: raise IndexError("Getting more than one result (%s) is not " @@ -158,11 +155,8 @@ def __setitem__(self, index, value): # `value` and assign to local arrays. This would dramatically # improve the fromndarray method's performance. - def setit(index, self_key, value): - main_module = __import__('__main__') - arr = reduce(getattr, [main_module] + self_key.split('.')) - res = arr.checked_setitem(index, value) - return res + def setit(arr, index, value): + return arr.checked_setitem(index, value) if isinstance(index, int) or isinstance(index, slice): tuple_index = (index,) @@ -170,9 +164,9 @@ def setit(index, self_key, value): elif isinstance(index, tuple): targets = self.distribution.owning_targets(index) - args = (index, self.key, value) - result = self.context.view._really_apply(setit, args=args, - targets=targets, block=True) + args = (self.key, index, value) + result = self.context.apply(setit, args=args, + targets=targets) result = [i for i in result if i is not None] if len(result) > 1: raise IndexError("Setting more than one result (%s) is not " From e53e43a957147e73b1a9180c53eb3c2cffa5ffcf Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 13 May 2014 14:18:25 -0500 Subject: [PATCH 5/5] Remove obsolete comment. --- distarray/dist/distarray.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/distarray/dist/distarray.py b/distarray/dist/distarray.py index de731dc3..7592a7a2 100644 --- a/distarray/dist/distarray.py +++ b/distarray/dist/distarray.py @@ -132,8 +132,6 @@ def getit(arr, index): elif isinstance(index, tuple): targets = self.distribution.owning_targets(index) - # Using a private IPython function is probably a bad think to do. - # but its the only apply function that lets us specify `targets`. args = (self.key, index) result = self.context.apply(getit, args=args, targets=targets)