From 15408195b0a4480dda972c354338a2412e95e2e7 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 10 Jun 2014 15:55:32 -0500 Subject: [PATCH 1/4] Reduce get_ndarray communication. --- distarray/dist/distarray.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/distarray/dist/distarray.py b/distarray/dist/distarray.py index 25294c25..bf08a930 100644 --- a/distarray/dist/distarray.py +++ b/distarray/dist/distarray.py @@ -323,11 +323,9 @@ def get_ndarrays(self): one ndarray per process """ - key = self.context._generate_key() - self.context._execute('%s = %s.get_localarray()' % (key, self.key), - targets=self.targets) - result = self.context._pull(key, targets=self.targets) - return result + def get(key): + return key.get_localarray() + return self.context.apply(get, args=(self.key,), targets=self.targets) def get_localarrays(self): """Pull the LocalArray objects from the engines. From 76f7bb35ba01d7b452868ee8e0fdf3eb94418f29 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 10 Jun 2014 16:08:28 -0500 Subject: [PATCH 2/4] Make get_localarrays use apply. And use `get_localarrays`. --- distarray/dist/distarray.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/distarray/dist/distarray.py b/distarray/dist/distarray.py index bf08a930..a3e53c92 100644 --- a/distarray/dist/distarray.py +++ b/distarray/dist/distarray.py @@ -245,9 +245,7 @@ def targets(self): def tondarray(self): """Returns the distributed array as an ndarray.""" arr = np.empty(self.shape, dtype=self.dtype) - local_name = self.context._generate_key() - self.context._execute('%s = %s.copy()' % (local_name, self.key), targets=self.targets) - local_arrays = self.context._pull(local_name, targets=self.targets) + local_arrays = self.get_localarrays() for local_array in local_arrays: maps = (list(ax_map.global_iter) for ax_map in local_array.distribution) @@ -336,8 +334,9 @@ def get_localarrays(self): one localarray per process """ - result = self.context._pull(self.key, targets=self.targets) - return result + def get(key): + return key.copy() + return self.context.apply(get, args=(self.key,), targets=self.targets) def get_localshapes(self): key = self.context._generate_key() From 4c888837b47fd775ae094608ade1c3db6e41fdd9 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Tue, 10 Jun 2014 16:12:19 -0500 Subject: [PATCH 3/4] Make get_localshapes use apply --- distarray/dist/distarray.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/distarray/dist/distarray.py b/distarray/dist/distarray.py index a3e53c92..fa079e07 100644 --- a/distarray/dist/distarray.py +++ b/distarray/dist/distarray.py @@ -339,11 +339,9 @@ def get(key): return self.context.apply(get, args=(self.key,), targets=self.targets) def get_localshapes(self): - key = self.context._generate_key() - self.context._execute('%s = %s.local_shape' % (key, self.key), - targets=self.targets) - result = self.context._pull(key, targets=self.targets) - return result + def get(key): + return key.local_shape + return self.context.apply(get, args=(self.key,), targets=self.targets) # Binary operators From 4de621106afdbf82f9aefff883ceb05964e23be5 Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Wed, 11 Jun 2014 10:53:56 -0500 Subject: [PATCH 4/4] Use apply in decorators.py type resolution. --- distarray/dist/decorators.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/distarray/dist/decorators.py b/distarray/dist/decorators.py index 32accfc9..62b7aaa5 100644 --- a/distarray/dist/decorators.py +++ b/distarray/dist/decorators.py @@ -130,10 +130,10 @@ def process_return_value(self, context, result_key): client and return it. If all but one of the pulled values is None, return that non-None value only. """ - type_key = context._generate_key() - type_statement = "{} = str(type({}))".format(type_key, result_key) - context._execute(type_statement, targets=context.targets) - result_type_str = context._pull(type_key, targets=context.targets) + def get_type_str(key): + return str(type(key)) + result_type_str = context.apply(get_type_str, args=(result_key,), + targets=context.targets) def is_NoneType(typestring): return (typestring == "" or